diff --git a/cmake/common/process/sparta_build_options.cmake b/cmake/common/process/sparta_build_options.cmake index 5aa48e28b..9d87742bc 100644 --- a/cmake/common/process/sparta_build_options.cmake +++ b/cmake/common/process/sparta_build_options.cmake @@ -199,6 +199,15 @@ endif() set(SPARTA_DEFAULT_CXX_COMPILE_FLAGS -DSPARTA_KOKKOS_EXACT ${SPARTA_DEFAULT_CXX_COMPILE_FLAGS}) endif() + # SPARTA_KOKKOS_FIXED_LISTS restores the fixed-size KKCopy arrays for the + # per-type tally compute lists instead of runtime-sized device buffers. The + # buffers lift the instance caps; the arrays keep every compute inside the + # kernel functor. Which performs better is hardware-dependent -- functor + # size trades against occupancy and data locality -- so both are buildable. + if(SPARTA_KOKKOS_FIXED_LISTS) + set(SPARTA_DEFAULT_CXX_COMPILE_FLAGS -DSPARTA_KOKKOS_FIXED_LISTS + ${SPARTA_DEFAULT_CXX_COMPILE_FLAGS}) + endif() # PKG_KOKKOS depends on BUILD_KOKKOS set(BUILD_KOKKOS ON) endif() diff --git a/cmake/common/set/sparta_cmake_defaults.cmake b/cmake/common/set/sparta_cmake_defaults.cmake index 1774ccc6c..59afc947f 100644 --- a/cmake/common/set/sparta_cmake_defaults.cmake +++ b/cmake/common/set/sparta_cmake_defaults.cmake @@ -49,6 +49,10 @@ if(SPARTA_ENABLE_TESTING) "surf_react_heatflux" "chem_rates" "custom" + "tally_computes" + "ambi_3body" + "regions" + "qk" "explicit2implicit" "mfp_mct" "optmove" @@ -89,9 +93,6 @@ if(SPARTA_ENABLE_TESTING) # the non-KOKKOS configurations. if(SPARTA_KOKKOS_EXACT) list(APPEND SPARTA_DISABLED_TESTS - # external field fix not KOKKOS-enabled - "in.bfield" - "in.bfield.grid" # VTK dump styles have no KOKKOS variant "in.vtk" "in.vtk.3d" diff --git a/doc/Section_accelerate.txt b/doc/Section_accelerate.txt index c4aae94f9..6173780b4 100644 --- a/doc/Section_accelerate.txt +++ b/doc/Section_accelerate.txt @@ -530,6 +530,35 @@ non-Kokkos fix or compute, or performing I/O for "stat"_stats.html or "dump"_dump.html output will cause data to be copied back to the CPU incurring a performance penalty. +NOTE: Most non-Kokkos styles degrade this way, costing performance but +still producing correct results. A few, however, are rejected outright +and will stop the run, rather than falling back to the host. Those +restrictions are documented on the page for the style that imposes them. + +NOTE: A run may define any number of instances of the styles that the +device kernels capture -- "surf_collide"_surf_collide.html, +"surf_react"_surf_react.html, and the per-event tally computes such as +"compute boundary"_compute_boundary.html, "compute +surf"_compute_surf.html, "compute isurf/grid"_compute_isurf_grid.html, +"compute react/boundary"_compute_react_boundary.html, "compute +react/surf"_compute_react_surf.html and "compute +react/isurf/grid"_compute_react_isurf_grid.html. Each is held in a +runtime-sized device buffer, so there is no compile-time cap. The one +remaining exception is that at most two "compute +surf"_compute_surf.html instances may tally for a single "fix +emit/surf"_fix_emit_surf.html; exceeding that stops the run with an +explanatory message. + +NOTE: Building with -DSPARTA_KOKKOS_FIXED_LISTS instead holds those +styles in fixed-size arrays inside the kernel functor, which restores +the former caps: at most two instances of each +"surf_react"_surf_react.html style, at most four surf react models in +total, at most two active instances of each per-event tally compute +listed above, and at most four gas-phase tally computes. The two +layouts produce identical results; which is faster is a +hardware-dependent question, so both are kept so they can be compared +by rebuilding. + [Run with the KOKKOS package by editing an input script:] Alternatively the effect of the "-sf" or "-pk" switches can be diff --git a/doc/Section_commands.txt b/doc/Section_commands.txt index 1844306a6..a4b905742 100644 --- a/doc/Section_commands.txt +++ b/doc/Section_commands.txt @@ -326,7 +326,7 @@ section"_#cmd_4 lists many of the same commands, grouped by category. "create_box"_create_box.html, "create_grid"_create_grid.html, "create_isurf"_create_isurf.html, -"create_particles"_create_particles.html, +"create_particles (k)"_create_particles.html, "custom"_custom.html, "dimension"_dimension.html, "dump"_dump.html, @@ -357,9 +357,9 @@ section"_#cmd_4 lists many of the same commands, grouped by category. "read_isurf"_read_isurf.html, "read_particles"_read_particles.html, "read_restart"_read_restart.html, -"read_surf"_read_surf.html, +"read_surf (k)"_read_surf.html, "region"_region.html, -"remove_surf"_remove_surf.html, +"remove_surf (k)"_remove_surf.html, "reset_timestep"_reset_timestep.html, "restart"_restart.html, "run"_run.html, @@ -410,14 +410,14 @@ This is indicated by additional letters in parenthesis: k = KOKKOS. "dt/reset (k)"_fix_dt_reset.html, "emit/face (k)"_fix_emit_face.html, "emit/face/file"_fix_emit_face_file.html, -"emit/surf"_fix_emit_surf.html, -"field/grid"_fix_field_grid.html, -"field/particle"_fix_field_particle.html, +"emit/surf (k)"_fix_emit_surf.html, +"field/grid (k)"_fix_field_grid.html, +"field/particle (k)"_fix_field_particle.html, "grid/check (k)"_fix_grid_check.html, "halt"_fix_halt.html, "move/surf (k)"_fix_move_surf.html, "print"_fix_print.html, -"surf/temp"_fix_surf_temp.html, +"surf/temp (k)"_fix_surf_temp.html, "temp/global/rescale (k)"_fix_temp_global_rescale.html, "temp/rescale (k)"_fix_temp_rescale.html, "vibmode (k)"_fix_vibmode.html :tb(c=6,ea=c) @@ -440,9 +440,9 @@ letters in parenthesis: k = KOKKOS. "eflux/grid (k)"_compute_eflux_grid.html, "fft/grid (k)"_compute_fft_grid.html, "gas/collision/grid (k)"_compute_gas_collision_grid.html, -"gas/collision/tally"_compute_gas_collision_tally.html, +"gas/collision/tally (k)"_compute_gas_collision_tally.html, "gas/reaction/grid (k)"_compute_gas_reaction_grid.html, -"gas/reaction/tally"_compute_gas_reaction_tally.html, +"gas/reaction/tally (k)"_compute_gas_reaction_tally.html, "grid (k)"_compute_grid.html, "isurf/grid (k)"_compute_isurf_grid.html, "ke/particle (k)"_compute_ke_particle.html, @@ -450,14 +450,14 @@ letters in parenthesis: k = KOKKOS. "pflux/grid (k)"_compute_pflux_grid.html, "property/grid (k)"_compute_property_grid.html, "property/surf (k)"_compute_property_surf.html, -"react/boundary"_compute_react_boundary.html, +"react/boundary (k)"_compute_react_boundary.html, "react/surf (k)"_compute_react_surf.html, "react/isurf/grid (k)"_compute_react_isurf_grid.html, -"reduce"_compute_reduce.html, +"reduce (k)"_compute_reduce.html, "sonine/grid (k)"_compute_sonine_grid.html, "surf (k)"_compute_surf.html, -"surf/collision/tally"_compute_surf_collision_tally.html, -"surf/reaction/tally"_compute_surf_reaction_tally.html, +"surf/collision/tally (k)"_compute_surf_collision_tally.html, +"surf/reaction/tally (k)"_compute_surf_reaction_tally.html, "temp (k)"_compute_temp.html, "thermal/grid (k)"_compute_thermal_grid.html, "tvib/grid (k)"_compute_tvib_grid.html :tb(c=6,ea=c) diff --git a/doc/compute_gas_collision_grid.txt b/doc/compute_gas_collision_grid.txt index ff66b611f..48f04f678 100644 --- a/doc/compute_gas_collision_grid.txt +++ b/doc/compute_gas_collision_grid.txt @@ -55,6 +55,29 @@ for an overview of SPARTA output options. :line +:line + +Styles with a {kk} suffix are functionally the same as the +corresponding style without the suffix. They have been optimized to +run faster, depending on your available hardware, as discussed in the +"Accelerating SPARTA"_Section_accelerate.html section of the manual. +The accelerated styles take the same arguments and should produce the +same results, except for different random number, round-off and +precision issues. + +These accelerated styles are part of the KOKKOS package. They are only +enabled if SPARTA was built with that package. See the "Making +SPARTA"_Section_start.html#start_3 section for more info. + +You can specify the accelerated styles explicitly in your input script +by including their suffix, or you can use the "-suffix command-line +switch"_Section_start.html#start_7 when you invoke SPARTA, or you can +use the "suffix"_suffix.html command in your input script. + +See the "Accelerating SPARTA"_Section_accelerate.html section of the +manual for more instructions on how to use the accelerated styles +effectively. + [Restrictions:] none [Related commands:] diff --git a/doc/compute_gas_collision_tally.txt b/doc/compute_gas_collision_tally.txt index 13d05da77..39ac7e18c 100644 --- a/doc/compute_gas_collision_tally.txt +++ b/doc/compute_gas_collision_tally.txt @@ -96,6 +96,29 @@ velocity keywords are in velocity units. :line +:line + +Styles with a {kk} suffix are functionally the same as the +corresponding style without the suffix. They have been optimized to +run faster, depending on your available hardware, as discussed in the +"Accelerating SPARTA"_Section_accelerate.html section of the manual. +The accelerated styles take the same arguments and should produce the +same results, except for different random number, round-off and +precision issues. + +These accelerated styles are part of the KOKKOS package. They are only +enabled if SPARTA was built with that package. See the "Making +SPARTA"_Section_start.html#start_3 section for more info. + +You can specify the accelerated styles explicitly in your input script +by including their suffix, or you can use the "-suffix command-line +switch"_Section_start.html#start_7 when you invoke SPARTA, or you can +use the "suffix"_suffix.html command in your input script. + +See the "Accelerating SPARTA"_Section_accelerate.html section of the +manual for more instructions on how to use the accelerated styles +effectively. + [Restrictions:] none [Related commands:] diff --git a/doc/compute_gas_reaction_grid.txt b/doc/compute_gas_reaction_grid.txt index 5a4bb9b53..09adda096 100644 --- a/doc/compute_gas_reaction_grid.txt +++ b/doc/compute_gas_reaction_grid.txt @@ -95,6 +95,29 @@ options. :line +:line + +Styles with a {kk} suffix are functionally the same as the +corresponding style without the suffix. They have been optimized to +run faster, depending on your available hardware, as discussed in the +"Accelerating SPARTA"_Section_accelerate.html section of the manual. +The accelerated styles take the same arguments and should produce the +same results, except for different random number, round-off and +precision issues. + +These accelerated styles are part of the KOKKOS package. They are only +enabled if SPARTA was built with that package. See the "Making +SPARTA"_Section_start.html#start_3 section for more info. + +You can specify the accelerated styles explicitly in your input script +by including their suffix, or you can use the "-suffix command-line +switch"_Section_start.html#start_7 when you invoke SPARTA, or you can +use the "suffix"_suffix.html command in your input script. + +See the "Accelerating SPARTA"_Section_accelerate.html section of the +manual for more instructions on how to use the accelerated styles +effectively. + [Restrictions:] none [Related commands:] diff --git a/doc/compute_gas_reaction_tally.txt b/doc/compute_gas_reaction_tally.txt index a3ae62707..4f1c0a2c1 100644 --- a/doc/compute_gas_reaction_tally.txt +++ b/doc/compute_gas_reaction_tally.txt @@ -130,6 +130,29 @@ velocity components are in velocity units. :line +:line + +Styles with a {kk} suffix are functionally the same as the +corresponding style without the suffix. They have been optimized to +run faster, depending on your available hardware, as discussed in the +"Accelerating SPARTA"_Section_accelerate.html section of the manual. +The accelerated styles take the same arguments and should produce the +same results, except for different random number, round-off and +precision issues. + +These accelerated styles are part of the KOKKOS package. They are only +enabled if SPARTA was built with that package. See the "Making +SPARTA"_Section_start.html#start_3 section for more info. + +You can specify the accelerated styles explicitly in your input script +by including their suffix, or you can use the "-suffix command-line +switch"_Section_start.html#start_7 when you invoke SPARTA, or you can +use the "suffix"_suffix.html command in your input script. + +See the "Accelerating SPARTA"_Section_accelerate.html section of the +manual for more instructions on how to use the accelerated styles +effectively. + [Restrictions:] none [Related commands:] diff --git a/doc/compute_react_boundary.txt b/doc/compute_react_boundary.txt index 0f4352d26..d39f7c7d2 100644 --- a/doc/compute_react_boundary.txt +++ b/doc/compute_react_boundary.txt @@ -7,6 +7,7 @@ :line compute react/boundary command :h3 +compute react/boundary/kk command :h3 [Syntax:] @@ -88,6 +89,29 @@ on each face. :line +:line + +Styles with a {kk} suffix are functionally the same as the +corresponding style without the suffix. They have been optimized to +run faster, depending on your available hardware, as discussed in the +"Accelerating SPARTA"_Section_accelerate.html section of the manual. +The accelerated styles take the same arguments and should produce the +same results, except for different random number, round-off and +precision issues. + +These accelerated styles are part of the KOKKOS package. They are only +enabled if SPARTA was built with that package. See the "Making +SPARTA"_Section_start.html#start_3 section for more info. + +You can specify the accelerated styles explicitly in your input script +by including their suffix, or you can use the "-suffix command-line +switch"_Section_start.html#start_7 when you invoke SPARTA, or you can +use the "suffix"_suffix.html command in your input script. + +See the "Accelerating SPARTA"_Section_accelerate.html section of the +manual for more instructions on how to use the accelerated styles +effectively. + [Restrictions:] none [Related commands:] diff --git a/doc/compute_reduce.txt b/doc/compute_reduce.txt index ace6e947b..d5ee4a4fc 100644 --- a/doc/compute_reduce.txt +++ b/doc/compute_reduce.txt @@ -7,6 +7,7 @@ :line compute reduce command :h3 +compute reduce/kk command :h3 [Syntax:] @@ -263,6 +264,38 @@ overview of SPARTA output options. The scalar or vector values will be in whatever "units"_units.html the quantities being reduced are in. +:line + +Styles with a {kk} suffix are functionally the same as the +corresponding style without the suffix. They have been optimized to +run faster, depending on your available hardware, as discussed in the +"Accelerating SPARTA"_Section_accelerate.html section of the manual. +The accelerated styles take the same arguments and should produce the +same results, except for different random number, round-off and +precision issues. + +These accelerated styles are part of the KOKKOS package. They are only +enabled if SPARTA was built with that package. See the "Making +SPARTA"_Section_start.html#start_3 section for more info. + +For the {kk} style, an input is reduced on the device when its source is +device-resident: the explicit per-particle attributes ({x}, {vx}, {ke}, +{erot}, {evib}, etc), per-particle and per-grid custom attributes, and +the per-particle or per-grid output of a compute or fix that itself has +a {kk} variant. Every other input -- per-surf inputs, {v_name} +variables, and the output of a non-Kokkos compute or fix -- is reduced +on the host, after the data it reads is copied back. This is a +performance distinction only; the result is the same either way. + +You can specify the accelerated styles explicitly in your input script +by including their suffix, or you can use the "-suffix command-line +switch"_Section_start.html#start_7 when you invoke SPARTA, or you can +use the "suffix"_suffix.html command in your input script. + +See the "Accelerating SPARTA"_Section_accelerate.html section of the +manual for more instructions on how to use the accelerated styles +effectively. + [Restrictions:] none [Related commands:] diff --git a/doc/compute_surf_collision_tally.txt b/doc/compute_surf_collision_tally.txt index 19f86f238..737865120 100644 --- a/doc/compute_surf_collision_tally.txt +++ b/doc/compute_surf_collision_tally.txt @@ -112,6 +112,29 @@ are {vx/post}, {vy/post}, {vz/post}. :line +:line + +Styles with a {kk} suffix are functionally the same as the +corresponding style without the suffix. They have been optimized to +run faster, depending on your available hardware, as discussed in the +"Accelerating SPARTA"_Section_accelerate.html section of the manual. +The accelerated styles take the same arguments and should produce the +same results, except for different random number, round-off and +precision issues. + +These accelerated styles are part of the KOKKOS package. They are only +enabled if SPARTA was built with that package. See the "Making +SPARTA"_Section_start.html#start_3 section for more info. + +You can specify the accelerated styles explicitly in your input script +by including their suffix, or you can use the "-suffix command-line +switch"_Section_start.html#start_7 when you invoke SPARTA, or you can +use the "suffix"_suffix.html command in your input script. + +See the "Accelerating SPARTA"_Section_accelerate.html section of the +manual for more instructions on how to use the accelerated styles +effectively. + [Restrictions:] none [Related commands:] diff --git a/doc/compute_surf_reaction_tally.txt b/doc/compute_surf_reaction_tally.txt index fe735cc4a..3e2a279bb 100644 --- a/doc/compute_surf_reaction_tally.txt +++ b/doc/compute_surf_reaction_tally.txt @@ -151,6 +151,29 @@ as are {vx1/post}, {vy1/post}, {vz1/post} and {vx2/post}, {vy2/post}, :line +:line + +Styles with a {kk} suffix are functionally the same as the +corresponding style without the suffix. They have been optimized to +run faster, depending on your available hardware, as discussed in the +"Accelerating SPARTA"_Section_accelerate.html section of the manual. +The accelerated styles take the same arguments and should produce the +same results, except for different random number, round-off and +precision issues. + +These accelerated styles are part of the KOKKOS package. They are only +enabled if SPARTA was built with that package. See the "Making +SPARTA"_Section_start.html#start_3 section for more info. + +You can specify the accelerated styles explicitly in your input script +by including their suffix, or you can use the "-suffix command-line +switch"_Section_start.html#start_7 when you invoke SPARTA, or you can +use the "suffix"_suffix.html command in your input script. + +See the "Accelerating SPARTA"_Section_accelerate.html section of the +manual for more instructions on how to use the accelerated styles +effectively. + [Restrictions:] none [Related commands:] diff --git a/doc/compute_tvib_grid.txt b/doc/compute_tvib_grid.txt index 2f083d303..e32bf5019 100644 --- a/doc/compute_tvib_grid.txt +++ b/doc/compute_tvib_grid.txt @@ -170,6 +170,29 @@ for an overview of SPARTA output options. The per-grid array values will be in temperature "units"_units.html. +:line + +Styles with a {kk} suffix are functionally the same as the +corresponding style without the suffix. They have been optimized to +run faster, depending on your available hardware, as discussed in the +"Accelerating SPARTA"_Section_accelerate.html section of the manual. +The accelerated styles take the same arguments and should produce the +same results, except for different random number, round-off and +precision issues. + +These accelerated styles are part of the KOKKOS package. They are only +enabled if SPARTA was built with that package. See the "Making +SPARTA"_Section_start.html#start_3 section for more info. + +You can specify the accelerated styles explicitly in your input script +by including their suffix, or you can use the "-suffix command-line +switch"_Section_start.html#start_7 when you invoke SPARTA, or you can +use the "suffix"_suffix.html command in your input script. + +See the "Accelerating SPARTA"_Section_accelerate.html section of the +manual for more instructions on how to use the accelerated styles +effectively. + [Restrictions:] none [Related commands:] diff --git a/doc/fix_dt_reset.txt b/doc/fix_dt_reset.txt index 4bc8fc8c1..9192f318b 100644 --- a/doc/fix_dt_reset.txt +++ b/doc/fix_dt_reset.txt @@ -101,6 +101,29 @@ It also computes a global vector of length 3 with these values: 2 = DTmax 3 = DTave :ul +:line + +Styles with a {kk} suffix are functionally the same as the +corresponding style without the suffix. They have been optimized to +run faster, depending on your available hardware, as discussed in the +"Accelerating SPARTA"_Section_accelerate.html section of the manual. +The accelerated styles take the same arguments and should produce the +same results, except for different random number, round-off and +precision issues. + +These accelerated styles are part of the KOKKOS package. They are only +enabled if SPARTA was built with that package. See the "Making +SPARTA"_Section_start.html#start_3 section for more info. + +You can specify the accelerated styles explicitly in your input script +by including their suffix, or you can use the "-suffix command-line +switch"_Section_start.html#start_7 when you invoke SPARTA, or you can +use the "suffix"_suffix.html command in your input script. + +See the "Accelerating SPARTA"_Section_accelerate.html section of the +manual for more instructions on how to use the accelerated styles +effectively. + [Related commands:] "compute dt/grid"_compute_dt_grid.html diff --git a/doc/fix_emit_face_file.txt b/doc/fix_emit_face_file.txt index 595ce22c1..a426e3c8b 100644 --- a/doc/fix_emit_face_file.txt +++ b/doc/fix_emit_face_file.txt @@ -19,11 +19,12 @@ face = {xlo} or {xhi} or {ylo} or {yhi} or {zlo} or {zhi} :l filename = input data file with boundary values for the emission :l boundary-ID = section of data file to read :l zero or more keyword/value pairs may be appended :l -keyword = {frac} or {nevery} or {perspecies} or {region} :l +keyword = {frac} or {nevery} or {perspecies} or {region} or {twopass} :l {frac} value = fraction = 0.0 to 1.0 fraction of particles to insert {nevery} value = Nstep = insert every this many timesteps {perspecies} value = {yes} or {no} - {region} value = region-ID :pre + {region} value = region-ID + {twopass} values = none :pre :ule [Examples:] @@ -355,6 +356,16 @@ that the {side} option for the "region"_region.html command can be used to define whether the inside or outside of the geometric region is considered to be "in" the region. +The {twopass} keyword does not require a value. If used, the +insertion procedure will loop over the insertion grid cells twice, the +same as the KOKKOS package version of this fix does, so that it can +reallocate memory efficiently, e.g. on a GPU. If this keyword is used +the non-KOKKOS and KOKKOS version will generate exactly the same set +of particles, which makes debugging easier. If the keyword is not +used, the non-KOKKOS and KOKKOS runs will use random numbers +differently and thus generate different particles, though they will be +statistically similar. + :line [Restart, output info:] @@ -395,7 +406,7 @@ emit/face"_fix_emit_face.html [Default:] The keyword defaults are frac = 1.0, nevery = 1, perspecies = yes, -region = none. +region = none, no twopass setting. :line diff --git a/doc/fix_emit_surf.txt b/doc/fix_emit_surf.txt index 8ec6e7998..a00c00fdc 100644 --- a/doc/fix_emit_surf.txt +++ b/doc/fix_emit_surf.txt @@ -17,7 +17,7 @@ emit/surf = style name of this fix command :l mix-ID = ID of mixture to use when creating particles :l group-ID = ID of surface group that emits particles :l zero or more keyword/value pairs may be appended :l -keyword = {n} or {normal} or {nevery} or {perspecies} or {region} or {subsonic} or {mflow} or {custom} :l +keyword = {n} or {normal} or {nevery} or {perspecies} or {region} or {subsonic} or {mflow} or {custom} or {twopass} :l {n} value = Np = number of particles to create Np can be a variable (see below) {normal} value = yes or no = emit normal to surface elements or with streaming velocity @@ -36,7 +36,8 @@ keyword = {n} or {normal} or {nevery} or {perspecies} or {region} or {subsonic} {window} value = Nwin = moving-average window for the cell streaming velocity {custom} values = attribute s_name attribute = {density} or {temperature} or {vstream} or {speed} or {fractions} - s_name = custom per-surf vector or array with name :pre + s_name = custom per-surf vector or array with name + {twopass} values = none :pre :ule [Examples:] @@ -417,6 +418,16 @@ mixture. This is determined by the "mixture"_mixture.html command. It is the order the gas species names were listed when the mixture command was specified (one or more times). +The {twopass} keyword does not require a value. If used, the +insertion procedure will loop over the insertion surface elements +twice, the same as the KOKKOS package version of this fix does, so +that it can reallocate memory efficiently, e.g. on a GPU. If this +keyword is used the non-KOKKOS and KOKKOS version will generate +exactly the same set of particles, which makes debugging easier. If +the keyword is not used, the non-KOKKOS and KOKKOS runs will use +random numbers differently and thus generate different particles, +though they will be statistically similar. + :line [Restart, output info:] @@ -431,6 +442,29 @@ second element is the cummulative total number added since the beginning of the run. The 2nd value is initialized to zero each time a run is performed. +:line + +Styles with a {kk} suffix are functionally the same as the +corresponding style without the suffix. They have been optimized to +run faster, depending on your available hardware, as discussed in the +"Accelerating SPARTA"_Section_accelerate.html section of the manual. +The accelerated styles take the same arguments and should produce the +same results, except for different random number, round-off and +precision issues. + +These accelerated styles are part of the KOKKOS package. They are only +enabled if SPARTA was built with that package. See the "Making +SPARTA"_Section_start.html#start_3 section for more info. + +You can specify the accelerated styles explicitly in your input script +by including their suffix, or you can use the "-suffix command-line +switch"_Section_start.html#start_7 when you invoke SPARTA, or you can +use the "suffix"_suffix.html command in your input script. + +See the "Accelerating SPARTA"_Section_accelerate.html section of the +manual for more instructions on how to use the accelerated styles +effectively. + [Restrictions:] A {n} setting of {Np} > 0 or {Np} as a variable can only be used with @@ -455,9 +489,9 @@ emit/face"_fix_emit_face.html [Default:] The keyword defaults are n = 0, normal = no, nevery = 1, perspecies = -yes, region = none, no subsonic settings, no mflow settings. For the -{subsonic} and {mflow} keywords, the moving-average window defaults to -0 (no smoothing). +yes, region = none, no subsonic settings, no mflow settings, no +twopass setting. For the {subsonic} and {mflow} keywords, the +moving-average window defaults to 0 (no smoothing). :line diff --git a/doc/fix_field_grid.txt b/doc/fix_field_grid.txt index ddbf01952..2ea238834 100644 --- a/doc/fix_field_grid.txt +++ b/doc/fix_field_grid.txt @@ -108,6 +108,29 @@ the grid-style variables. The number of rows in the array is the number of grid cells this processor owns. The number of columns in the array is the number of non-NULL variables specified. +:line + +Styles with a {kk} suffix are functionally the same as the +corresponding style without the suffix. They have been optimized to +run faster, depending on your available hardware, as discussed in the +"Accelerating SPARTA"_Section_accelerate.html section of the manual. +The accelerated styles take the same arguments and should produce the +same results, except for different random number, round-off and +precision issues. + +These accelerated styles are part of the KOKKOS package. They are only +enabled if SPARTA was built with that package. See the "Making +SPARTA"_Section_start.html#start_3 section for more info. + +You can specify the accelerated styles explicitly in your input script +by including their suffix, or you can use the "-suffix command-line +switch"_Section_start.html#start_7 when you invoke SPARTA, or you can +use the "suffix"_suffix.html command in your input script. + +See the "Accelerating SPARTA"_Section_accelerate.html section of the +manual for more instructions on how to use the accelerated styles +effectively. + [Restrictions:] none [Related commands:] diff --git a/doc/fix_field_particle.txt b/doc/fix_field_particle.txt index b3c088137..8bb8b1c2d 100644 --- a/doc/fix_field_particle.txt +++ b/doc/fix_field_particle.txt @@ -100,6 +100,29 @@ evaluating the particle-style variables. The number of rows in the array is the number of particles this processor owns. The number of columns in the array is the number of non-NULL variables specified. +:line + +Styles with a {kk} suffix are functionally the same as the +corresponding style without the suffix. They have been optimized to +run faster, depending on your available hardware, as discussed in the +"Accelerating SPARTA"_Section_accelerate.html section of the manual. +The accelerated styles take the same arguments and should produce the +same results, except for different random number, round-off and +precision issues. + +These accelerated styles are part of the KOKKOS package. They are only +enabled if SPARTA was built with that package. See the "Making +SPARTA"_Section_start.html#start_3 section for more info. + +You can specify the accelerated styles explicitly in your input script +by including their suffix, or you can use the "-suffix command-line +switch"_Section_start.html#start_7 when you invoke SPARTA, or you can +use the "suffix"_suffix.html command in your input script. + +See the "Accelerating SPARTA"_Section_accelerate.html section of the +manual for more instructions on how to use the accelerated styles +effectively. + [Restrictions:] none [Related commands:] diff --git a/doc/fix_surf_temp.txt b/doc/fix_surf_temp.txt index 974c99657..e30f65f60 100644 --- a/doc/fix_surf_temp.txt +++ b/doc/fix_surf_temp.txt @@ -123,6 +123,29 @@ However, the custom per-surf attribute defined by this fix can be accessed by the "dump surf"_dump.html command, as s_name. That means those per-surf values can be written to surface dump files. +:line + +Styles with a {kk} suffix are functionally the same as the +corresponding style without the suffix. They have been optimized to +run faster, depending on your available hardware, as discussed in the +"Accelerating SPARTA"_Section_accelerate.html section of the manual. +The accelerated styles take the same arguments and should produce the +same results, except for different random number, round-off and +precision issues. + +These accelerated styles are part of the KOKKOS package. They are only +enabled if SPARTA was built with that package. See the "Making +SPARTA"_Section_start.html#start_3 section for more info. + +You can specify the accelerated styles explicitly in your input script +by including their suffix, or you can use the "-suffix command-line +switch"_Section_start.html#start_7 when you invoke SPARTA, or you can +use the "suffix"_suffix.html command in your input script. + +See the "Accelerating SPARTA"_Section_accelerate.html section of the +manual for more instructions on how to use the accelerated styles +effectively. + [Restrictions:] This fix can only be used in simulations that define explicit diff --git a/doc/read_surf.txt b/doc/read_surf.txt index 6039bdb34..b278d8582 100644 --- a/doc/read_surf.txt +++ b/doc/read_surf.txt @@ -614,6 +614,29 @@ of the surface element IDs. :line :line +:line + +Styles with a {kk} suffix are functionally the same as the +corresponding style without the suffix. They have been optimized to +run faster, depending on your available hardware, as discussed in the +"Accelerating SPARTA"_Section_accelerate.html section of the manual. +The accelerated styles take the same arguments and should produce the +same results, except for different random number, round-off and +precision issues. + +These accelerated styles are part of the KOKKOS package. They are only +enabled if SPARTA was built with that package. See the "Making +SPARTA"_Section_start.html#start_3 section for more info. + +You can specify the accelerated styles explicitly in your input script +by including their suffix, or you can use the "-suffix command-line +switch"_Section_start.html#start_7 when you invoke SPARTA, or you can +use the "suffix"_suffix.html command in your input script. + +See the "Accelerating SPARTA"_Section_accelerate.html section of the +manual for more instructions on how to use the accelerated styles +effectively. + [Restrictions:] This command can only be used after the simulation box is defined by diff --git a/doc/region.txt b/doc/region.txt index 7ad8b3150..a7cbbe022 100644 --- a/doc/region.txt +++ b/doc/region.txt @@ -13,7 +13,7 @@ region command :h3 region ID style args keyword value ... :pre ID = user-assigned name for the region :ulb,l -style = {block} or {cylinder} or {plane} or {sphere} or {union} or {intersect} or {block/kk} or {cylinder/kk} or {plane/kk} or {sphere/kk} :l +style = {block} or {cylinder} or {plane} or {sphere} or {union} or {intersect} or {block/kk} or {cylinder/kk} or {plane/kk} or {sphere/kk} or {union/kk} or {intersect/kk} :l {block} or {block/kk} args = xlo xhi ylo yhi zlo zhi xlo,xhi,ylo,yhi,zlo,zhi = bounds of block in all dimensions (distance units) {cylinder} or {cylinder/kk} args = dim c1 c2 radius lo hi diff --git a/doc/remove_surf.txt b/doc/remove_surf.txt index 9a5730108..5f8e31944 100644 --- a/doc/remove_surf.txt +++ b/doc/remove_surf.txt @@ -37,6 +37,29 @@ elements have IDs from 1 to N. The new list of surface elements can be output via the "write_surf"_write_surf.html or "dump surf"_dump.html commands. +:line + +Styles with a {kk} suffix are functionally the same as the +corresponding style without the suffix. They have been optimized to +run faster, depending on your available hardware, as discussed in the +"Accelerating SPARTA"_Section_accelerate.html section of the manual. +The accelerated styles take the same arguments and should produce the +same results, except for different random number, round-off and +precision issues. + +These accelerated styles are part of the KOKKOS package. They are only +enabled if SPARTA was built with that package. See the "Making +SPARTA"_Section_start.html#start_3 section for more info. + +You can specify the accelerated styles explicitly in your input script +by including their suffix, or you can use the "-suffix command-line +switch"_Section_start.html#start_7 when you invoke SPARTA, or you can +use the "suffix"_suffix.html command in your input script. + +See the "Accelerating SPARTA"_Section_accelerate.html section of the +manual for more instructions on how to use the accelerated styles +effectively. + [Restrictions:] none [Related commands:] diff --git a/examples/ambi/in.ambi.group.react b/examples/ambi/in.ambi.group.react new file mode 100644 index 000000000..a623118e8 --- /dev/null +++ b/examples/ambi/in.ambi.group.react @@ -0,0 +1,50 @@ +################################################################################ +# thermal plasma in a 2d box, reacting multigroup ambipolar collisions +# +# Same setup as in.ambi.group, with gas-phase chemistry enabled and the +# temperature and density raised until reactions are frequent. This is the +# regression guard for the reacting multigroup ambipolar collision path, +# where group membership, the electron list and the cell particle list all +# change as reactions rebin, create and destroy particles. +# +# Note: +# - The "comm/sort" option to the "global" command is used to match MPI runs. +# - The "twopass" option is used to match Kokkos runs. +# The "comm/sort" and "twopass" options should not be used for production runs. +################################################################################ + +seed 12345 +dimension 2 +boundary rr rr p +global gridcut 0.01 comm/sort yes +create_box -2.0 2.0 -2.0 2.0 -0.5 0.5 +create_grid 20 20 1 +balance_grid rcb cell + +global fnum 2.6404E16 +global nrho 2.6404e23 + +species air.species N2 O2 N O NO N2+ O2+ N+ O+ NO+ e + +# collide mixture: all species, two groups +# the ambipolar electron species e must be in a group by itself + +mixture gas N2 O2 N O NO N2+ O2+ N+ O+ NO+ vstream 0 0 0 temp 100000.0 group heavy +mixture gas e group electron +mixture gas N2 frac 0.6 +mixture gas N2+ frac 0.4 + +fix ambi ambipolar e N+ N2+ NO+ O+ O2+ + +collide vss gas air.vss +collide_modify ambipolar yes +react tce air.tce + +create_particles gas n 10000 twopass + +compute temp temp +stats 25 +stats_style step np nattempt ncoll nreact c_temp + +timestep 1.0e-7 +run 300 diff --git a/examples/ambi/log.22Aug26.mpi_1.ambi.group.react b/examples/ambi/log.22Aug26.mpi_1.ambi.group.react new file mode 100644 index 000000000..ce5a55879 --- /dev/null +++ b/examples/ambi/log.22Aug26.mpi_1.ambi.group.react @@ -0,0 +1,140 @@ +SPARTA (24 Sep 2025) +Running on 1 MPI task(s) +################################################################################ +# thermal plasma in a 2d box, reacting multigroup ambipolar collisions +# +# Same setup as in.ambi.group, with gas-phase chemistry enabled and the +# temperature and density raised until reactions are frequent. This is the +# regression guard for the reacting multigroup ambipolar collision path, +# where group membership, the electron list and the cell particle list all +# change as reactions rebin, create and destroy particles. +# +# Note: +# - The "comm/sort" option to the "global" command is used to match MPI runs. +# - The "twopass" option is used to match Kokkos runs. +# The "comm/sort" and "twopass" options should not be used for production runs. +################################################################################ + +seed 12345 +dimension 2 +boundary rr rr p +global gridcut 0.01 comm/sort yes +create_box -2.0 2.0 -2.0 2.0 -0.5 0.5 +Created orthogonal box = (-2 -2 -0.5) to (2 2 0.5) +create_grid 20 20 1 +Created 400 child grid cells + CPU time = 0.00117751 secs + create/ghost percent = 88.9972 11.0028 +balance_grid rcb cell +Balance grid migrated 0 cells + CPU time = 0.00018752 secs + reassign/sort/migrate/ghost percent = 58.9857 0.617001 7.64505 32.7522 + +global fnum 2.6404E16 +global nrho 2.6404e23 + +species air.species N2 O2 N O NO N2+ O2+ N+ O+ NO+ e + +# collide mixture: all species, two groups +# the ambipolar electron species e must be in a group by itself + +mixture gas N2 O2 N O NO N2+ O2+ N+ O+ NO+ vstream 0 0 0 temp 100000.0 group heavy +mixture gas e group electron +mixture gas N2 frac 0.6 +mixture gas N2+ frac 0.4 + +fix ambi ambipolar e N+ N2+ NO+ O+ O2+ + +collide vss gas air.vss +collide_modify ambipolar yes +react tce air.tce + +create_particles gas n 10000 twopass +Created 10000 particles + CPU time = 0.0035726 secs + +compute temp temp +stats 25 +stats_style step np nattempt ncoll nreact c_temp + +timestep 1.0e-7 +run 300 +Memory usage per proc in Mbytes: + particles (ave,min,max) = 2 2 2 + grid (ave,min,max) = 1.51379 1.51379 1.51379 + surf (ave,min,max) = 0 0 0 + modify (ave,min,max) = 0 0 0 + total (ave,min,max) = 3.51379 3.51379 3.51379 +Step Np Natt Ncoll Nreact c_temp + 0 10000 0 0 0 100229.43 + 25 10002 3058 849 1 100241.89 +WARNING: TCE reaction probability exceeded 1.0, chemistry may be under-resolved, consider reducing timestep or fnum (further warnings suppressed) (/home/user/sparta/src/react_tce.cpp:204) + 50 10005 3062 841 0 100186.17 + 75 10013 3116 884 0 100032.28 + 100 10017 3150 841 2 99957.635 + 125 10021 3157 830 0 99935.586 + 150 10028 3157 853 0 99922.724 + 175 10034 3181 890 0 99857.183 + 200 10043 3206 839 0 99804.323 + 225 10050 3244 896 1 99696.68 + 250 10053 3233 891 3 99485.859 + 275 10062 3250 894 1 99459.755 + 300 10064 3230 836 1 99262.266 +Loop time of 0.196401 on 1 procs for 300 steps with 10064 particles +Performance: 1527.487 timesteps/s, 15.373 Mparticle-step/s + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Move | 0.02198 | 0.02198 | 0.02198 | 0.0 | 11.19 +Coll | 0.16691 | 0.16691 | 0.16691 | 0.0 | 84.99 +Sort | 0.0060744 | 0.0060744 | 0.0060744 | 0.0 | 3.09 +Comm | 0.00020011 | 0.00020011 | 0.00020011 | 0.0 | 0.10 +Modify | 0 | 0 | 0 | 0.0 | 0.00 +Output | 0.00098718 | 0.00098718 | 0.00098718 | 0.0 | 0.50 +MPI Sync| 0.00019926 | 0.00019926 | 0.00019926 | 0.0 | 0.10 +Other | | 4.504e-05 | | | 0.02 + +Particle moves = 3008980 (3.01M) +Cells touched = 3021456 (3.02M) +Particle comms = 0 (0K) +Boundary collides = 615 (0.615K) +Boundary exits = 0 (0K) +SurfColl checks = 0 (0K) +SurfColl occurs = 0 (0K) +Surf reactions = 0 (0K) +Collide attempts = 949017 (0.949M) +Collide occurs = 257717 (0.258M) +Reactions = 169 (0.169K) +Particles stuck = 0 +Axisymm bad moves = 0 + +Particle-moves/CPUsec/proc: 1.53206e+07 +Particle-moves/step: 10029.9 +Cell-touches/particle/step: 1.00415 +Particle comm iterations/step: 1 +Particle fraction communicated: 0 +Particle fraction colliding with boundary: 0.000204388 +Particle fraction exiting boundary: 0 +Surface-checks/particle/step: 0 +Surface-collisions/particle/step: 0 +Surf-reactions/particle/step: 0 +Collision-attempts/particle/step: 0.315395 +Collisions/particle/step: 0.0856493 +Reactions/particle/step: 5.61652e-05 + +Gas reaction tallies: + style tce #-of-reactions 45 + reaction N2 + N2 --> N + N + N2: 64 + reaction N2 + N+ --> N + N2+: 2 + reaction N2+ + N --> N2 + N+: 2 + reaction N + e --> N+ + e + e: 101 + +Particles: 10064 ave 10064 max 10064 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Cells: 400 ave 400 max 400 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +GhostCell: 0 ave 0 max 0 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +EmptyCell: 0 ave 0 max 0 min +Histogram: 1 0 0 0 0 0 0 0 0 0 diff --git a/examples/ambi/log.22Aug26.mpi_4.ambi.group.react b/examples/ambi/log.22Aug26.mpi_4.ambi.group.react new file mode 100644 index 000000000..d6880eef8 --- /dev/null +++ b/examples/ambi/log.22Aug26.mpi_4.ambi.group.react @@ -0,0 +1,141 @@ +SPARTA (24 Sep 2025) +Running on 4 MPI task(s) +################################################################################ +# thermal plasma in a 2d box, reacting multigroup ambipolar collisions +# +# Same setup as in.ambi.group, with gas-phase chemistry enabled and the +# temperature and density raised until reactions are frequent. This is the +# regression guard for the reacting multigroup ambipolar collision path, +# where group membership, the electron list and the cell particle list all +# change as reactions rebin, create and destroy particles. +# +# Note: +# - The "comm/sort" option to the "global" command is used to match MPI runs. +# - The "twopass" option is used to match Kokkos runs. +# The "comm/sort" and "twopass" options should not be used for production runs. +################################################################################ + +seed 12345 +dimension 2 +boundary rr rr p +global gridcut 0.01 comm/sort yes +create_box -2.0 2.0 -2.0 2.0 -0.5 0.5 +Created orthogonal box = (-2 -2 -0.5) to (2 2 0.5) +create_grid 20 20 1 +WARNING: Could not acquire nearby ghost cells b/c grid partition is not clumped (/home/user/sparta/src/grid.cpp:486) +Created 400 child grid cells + CPU time = 0.0638546 secs + create/ghost percent = 12.3332 87.6668 +balance_grid rcb cell +Balance grid migrated 280 cells + CPU time = 0.363949 secs + reassign/sort/migrate/ghost percent = 56.0389 2.19864 10.9955 30.7669 + +global fnum 2.6404E16 +global nrho 2.6404e23 + +species air.species N2 O2 N O NO N2+ O2+ N+ O+ NO+ e + +# collide mixture: all species, two groups +# the ambipolar electron species e must be in a group by itself + +mixture gas N2 O2 N O NO N2+ O2+ N+ O+ NO+ vstream 0 0 0 temp 100000.0 group heavy +mixture gas e group electron +mixture gas N2 frac 0.6 +mixture gas N2+ frac 0.4 + +fix ambi ambipolar e N+ N2+ NO+ O+ O2+ + +collide vss gas air.vss +collide_modify ambipolar yes +react tce air.tce + +create_particles gas n 10000 twopass +Created 10000 particles + CPU time = 0.0188347 secs + +compute temp temp +stats 25 +stats_style step np nattempt ncoll nreact c_temp + +timestep 1.0e-7 +run 300 +Memory usage per proc in Mbytes: + particles (ave,min,max) = 2 2 2 + grid (ave,min,max) = 1.51379 1.51379 1.51379 + surf (ave,min,max) = 0 0 0 + modify (ave,min,max) = 0 0 0 + total (ave,min,max) = 3.51379 3.51379 3.51379 +Step Np Natt Ncoll Nreact c_temp + 0 10000 0 0 0 99791.345 + 25 10004 3101 852 0 99737.755 + 50 10005 3123 848 0 99808.067 + 75 10011 3140 900 2 99739.046 +WARNING: TCE reaction probability exceeded 1.0, chemistry may be under-resolved, consider reducing timestep or fnum (further warnings suppressed) (/home/user/sparta/src/react_tce.cpp:204) + 100 10014 3173 834 0 99632.539 + 125 10018 3185 861 0 99540.816 + 150 10027 3188 895 0 99532.743 + 175 10033 3206 869 0 99400.682 + 200 10039 3237 933 0 99495.122 + 225 10044 3236 876 0 99536.782 + 250 10053 3240 853 1 99457.24 + 275 10059 3239 870 1 99420.092 + 300 10063 3295 927 1 99327.363 +Loop time of 6.77107 on 4 procs for 300 steps with 10063 particles +Performance: 44.306 timesteps/s, 445.853 kparticle-step/s + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Move | 0.0079257 | 0.0083554 | 0.0086259 | 0.3 | 0.12 +Coll | 0.053755 | 0.057922 | 0.060777 | 1.1 | 0.86 +Sort | 0.011579 | 0.012837 | 0.013665 | 0.7 | 0.19 +Comm | 3.9918 | 4.6744 | 5.5481 | 29.3 | 69.03 +Modify | 0 | 0 | 0 | 0.0 | 0.00 +Output | 0.39411 | 0.4104 | 0.42486 | 1.8 | 6.06 +MPI Sync| 0.75409 | 1.6071 | 2.287 | 49.5 | 23.73 +Other | | 9.051e-05 | | | 0.00 + +Particle moves = 3008498 (3.01M) +Cells touched = 3020945 (3.02M) +Particle comms = 654 (0.654K) +Boundary collides = 651 (0.651K) +Boundary exits = 0 (0K) +SurfColl checks = 0 (0K) +SurfColl occurs = 0 (0K) +Surf reactions = 0 (0K) +Collide attempts = 956834 (0.957M) +Collide occurs = 261003 (0.261M) +Reactions = 162 (0.162K) +Particles stuck = 0 +Axisymm bad moves = 0 + +Particle-moves/CPUsec/proc: 111079 +Particle-moves/step: 10028.3 +Cell-touches/particle/step: 1.00414 +Particle comm iterations/step: 1 +Particle fraction communicated: 0.000217384 +Particle fraction colliding with boundary: 0.000216387 +Particle fraction exiting boundary: 0 +Surface-checks/particle/step: 0 +Surface-collisions/particle/step: 0 +Surf-reactions/particle/step: 0 +Collision-attempts/particle/step: 0.318044 +Collisions/particle/step: 0.0867553 +Reactions/particle/step: 5.38475e-05 + +Gas reaction tallies: + style tce #-of-reactions 45 + reaction N2 + N2 --> N + N + N2: 63 + reaction N2 + N+ --> N + N2+: 4 + reaction N2+ + N --> N2 + N+: 1 + reaction N + e --> N+ + e + e: 94 + +Particles: 2515.75 ave 2543 max 2504 min +Histogram: 1 2 0 0 0 0 0 0 0 1 +Cells: 100 ave 100 max 100 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +GhostCell: 21 ave 21 max 21 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +EmptyCell: 0 ave 0 max 0 min +Histogram: 4 0 0 0 0 0 0 0 0 0 diff --git a/examples/ambi_3body/in.ambi_3body b/examples/ambi_3body/in.ambi_3body index 6d5911afd..1c7676c63 100644 --- a/examples/ambi_3body/in.ambi_3body +++ b/examples/ambi_3body/in.ambi_3body @@ -6,10 +6,10 @@ # ion-third-body recombination, so any gas reactions that occur exercise the # new functionality directly. # -# A single-group mixture is used so the case runs on the KOKKOS package, -# which currently supports only single-group collisions. The "comm/sort" -# and "twopass" options make MPI and Kokkos runs reproducible and should not -# be used for production runs. +# A single-group mixture keeps the case focused on the third-body chemistry +# rather than on group binning; the KOKKOS package runs multigroup ambipolar +# collisions too. The "comm/sort" and "twopass" options make MPI and Kokkos +# runs reproducible and should not be used for production runs. ################################################################################ seed 12345 @@ -25,7 +25,7 @@ global nrho 1.0e22 fnum 1.0e8 species ambi_3body.species O2 O O2+ e -# collide mixture: all species in one group (required for Kokkos) +# collide mixture: all species in one group mixture plasma O2 O O2+ e temp 50000.0 # create mixture: heavy species only (electrons are added by fix ambipolar) diff --git a/examples/ambi_3body/log.22Aug26.mpi_1.ambi_3body b/examples/ambi_3body/log.22Aug26.mpi_1.ambi_3body new file mode 100644 index 000000000..2d30bf584 --- /dev/null +++ b/examples/ambi_3body/log.22Aug26.mpi_1.ambi_3body @@ -0,0 +1,139 @@ +SPARTA (24 Sep 2025) +Running on 1 MPI task(s) +################################################################################ +# Ambipolar ions as third bodies in dissociation and recombination (issue #176) +# +# A closed, periodic box of hot O2 / O / O2+ (+ ambipolar electrons). The +# only reactions defined are an ion-third-body dissociation and an +# ion-third-body recombination, so any gas reactions that occur exercise the +# new functionality directly. +# +# A single-group mixture keeps the case focused on the third-body chemistry +# rather than on group binning; the KOKKOS package runs multigroup ambipolar +# collisions too. The "comm/sort" and "twopass" options make MPI and Kokkos +# runs reproducible and should not be used for production runs. +################################################################################ + +seed 12345 +dimension 3 +global gridcut 0.0 comm/sort yes +boundary p p p + +create_box 0.0 0.001 0.0 0.001 0.0 0.001 +Created orthogonal box = (0 0 0) to (0.001 0.001 0.001) +create_grid 10 10 10 +Created 1000 child grid cells + CPU time = 0.00102249 secs + create/ghost percent = 80.2795 19.7205 +balance_grid rcb cell +Balance grid migrated 0 cells + CPU time = 0.000248273 secs + reassign/sort/migrate/ghost percent = 58.248 0.766092 8.32269 32.6632 + +global nrho 1.0e22 fnum 1.0e8 + +species ambi_3body.species O2 O O2+ e + +# collide mixture: all species in one group +mixture plasma O2 O O2+ e temp 50000.0 + +# create mixture: heavy species only (electrons are added by fix ambipolar) +mixture gas O2 O O2+ temp 50000.0 +mixture gas O2 frac 0.5 +mixture gas O frac 0.3 +mixture gas O2+ frac 0.2 + +fix ambi ambipolar e O2+ + +collide vss plasma ambi_3body.vss relax variable +collide_modify vremax 1000 yes vibrate discrete rotate smooth +collide_modify ambipolar yes +react tce ambi_3body.tce + +create_particles gas n 0 twopass +Created 100000 particles + CPU time = 0.0246759 secs + +timestep 1.0e-9 + +compute c count species +stats_style step cpu np nattempt ncoll nreact c_c[1] c_c[2] c_c[3] c_c[4] +stats 50 + +run 500 +WARNING: Single-group ambipolar collisions are inefficient; grouping electrons separately (e.g. collide ... species) is recommended (/home/user/sparta/src/collide.cpp:355) +Memory usage per proc in Mbytes: + particles (ave,min,max) = 14 14 14 + grid (ave,min,max) = 1.51379 1.51379 1.51379 + surf (ave,min,max) = 0 0 0 + modify (ave,min,max) = 0 0 0 + total (ave,min,max) = 15.5138 15.5138 15.5138 +Step CPU Np Natt Ncoll Nreact c_c[1] c_c[2] c_c[3] c_c[4] + 0 0 100000 0 0 0 50060 29971 19969 0 +WARNING: TCE reaction probability exceeded 1.0, chemistry may be under-resolved, consider reducing timestep or fnum (further warnings suppressed) (/home/user/sparta/src/react_tce.cpp:204) + 50 0.9342345 100468 221356 15197 10 49592 30907 19969 0 + 100 2.0531659 100922 223049 15298 16 49138 31815 19969 0 + 150 3.1847049 101389 224857 15076 15 48671 32749 19969 0 + 200 4.3356764 101838 226315 15131 9 48222 33647 19969 0 + 250 5.4944245 102295 228245 15188 9 47765 34561 19969 0 + 300 6.6389229 102713 229655 15235 11 47347 35397 19969 0 + 350 7.670481 103138 231217 15241 5 46922 36247 19969 0 + 400 8.7105795 103624 232840 15244 11 46436 37219 19969 0 + 450 9.775811 104053 234550 15135 6 46007 38077 19969 0 + 500 10.907967 104512 236322 15311 8 45548 38995 19969 0 +Loop time of 10.9081 on 1 procs for 500 steps with 104512 particles +Performance: 45.837 timesteps/s, 4.791 Mparticle-step/s + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Move | 0.54067 | 0.54067 | 0.54067 | 0.0 | 4.96 +Coll | 10.131 | 10.131 | 10.131 | 0.0 | 92.87 +Sort | 0.22906 | 0.22906 | 0.22906 | 0.0 | 2.10 +Comm | 0.0013194 | 0.0013194 | 0.0013194 | 0.0 | 0.01 +Modify | 0 | 0 | 0 | 0.0 | 0.00 +Output | 0.0049255 | 0.0049255 | 0.0049255 | 0.0 | 0.05 +MPI Sync| 0.0014474 | 0.0014474 | 0.0014474 | 0.0 | 0.01 +Other | | 0.0001644 | | | 0.00 + +Particle moves = 51133461 (51.1M) +Cells touched = 56122208 (56.1M) +Particle comms = 0 (0K) +Boundary collides = 0 (0K) +Boundary exits = 0 (0K) +SurfColl checks = 0 (0K) +SurfColl occurs = 0 (0K) +Surf reactions = 0 (0K) +Collide attempts = 113996640 (114M) +Collide occurs = 7623406 (7.62M) +Reactions = 4524 (4.52K) +Particles stuck = 0 +Axisymm bad moves = 0 + +Particle-moves/CPUsec/proc: 4.68766e+06 +Particle-moves/step: 102267 +Cell-touches/particle/step: 1.09756 +Particle comm iterations/step: 1 +Particle fraction communicated: 0 +Particle fraction colliding with boundary: 0 +Particle fraction exiting boundary: 0 +Surface-checks/particle/step: 0 +Surface-collisions/particle/step: 0 +Surf-reactions/particle/step: 0 +Collision-attempts/particle/step: 2.22939 +Collisions/particle/step: 0.149088 +Reactions/particle/step: 8.84744e-05 + +Gas reaction tallies: + style tce #-of-reactions 2 + reaction O2 + O2+ --> O + O2+ + O: 4518 + reaction O + O --> O2 + O2+: 6 + +Particles: 104512 ave 104512 max 104512 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Cells: 1000 ave 1000 max 1000 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +GhostCell: 0 ave 0 max 0 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +EmptyCell: 0 ave 0 max 0 min +Histogram: 1 0 0 0 0 0 0 0 0 0 diff --git a/examples/ambi_3body/log.22Aug26.mpi_4.ambi_3body b/examples/ambi_3body/log.22Aug26.mpi_4.ambi_3body new file mode 100644 index 000000000..02d146700 --- /dev/null +++ b/examples/ambi_3body/log.22Aug26.mpi_4.ambi_3body @@ -0,0 +1,140 @@ +SPARTA (24 Sep 2025) +Running on 4 MPI task(s) +################################################################################ +# Ambipolar ions as third bodies in dissociation and recombination (issue #176) +# +# A closed, periodic box of hot O2 / O / O2+ (+ ambipolar electrons). The +# only reactions defined are an ion-third-body dissociation and an +# ion-third-body recombination, so any gas reactions that occur exercise the +# new functionality directly. +# +# A single-group mixture keeps the case focused on the third-body chemistry +# rather than on group binning; the KOKKOS package runs multigroup ambipolar +# collisions too. The "comm/sort" and "twopass" options make MPI and Kokkos +# runs reproducible and should not be used for production runs. +################################################################################ + +seed 12345 +dimension 3 +global gridcut 0.0 comm/sort yes +boundary p p p + +create_box 0.0 0.001 0.0 0.001 0.0 0.001 +Created orthogonal box = (0 0 0) to (0.001 0.001 0.001) +create_grid 10 10 10 +WARNING: Could not acquire nearby ghost cells b/c grid partition is not clumped (/home/user/sparta/src/grid.cpp:486) +Created 1000 child grid cells + CPU time = 0.0010655 secs + create/ghost percent = 88.428 11.572 +balance_grid rcb cell +Balance grid migrated 740 cells + CPU time = 0.000843833 secs + reassign/sort/migrate/ghost percent = 43.6789 0.328501 15.9548 40.0378 + +global nrho 1.0e22 fnum 1.0e8 + +species ambi_3body.species O2 O O2+ e + +# collide mixture: all species in one group +mixture plasma O2 O O2+ e temp 50000.0 + +# create mixture: heavy species only (electrons are added by fix ambipolar) +mixture gas O2 O O2+ temp 50000.0 +mixture gas O2 frac 0.5 +mixture gas O frac 0.3 +mixture gas O2+ frac 0.2 + +fix ambi ambipolar e O2+ + +collide vss plasma ambi_3body.vss relax variable +collide_modify vremax 1000 yes vibrate discrete rotate smooth +collide_modify ambipolar yes +react tce ambi_3body.tce + +create_particles gas n 0 twopass +Created 99999 particles + CPU time = 0.0077028 secs + +timestep 1.0e-9 + +compute c count species +stats_style step cpu np nattempt ncoll nreact c_c[1] c_c[2] c_c[3] c_c[4] +stats 50 + +run 500 +WARNING: Single-group ambipolar collisions are inefficient; grouping electrons separately (e.g. collide ... species) is recommended (/home/user/sparta/src/collide.cpp:355) +Memory usage per proc in Mbytes: + particles (ave,min,max) = 4 4 4 + grid (ave,min,max) = 1.51379 1.51379 1.51379 + surf (ave,min,max) = 0 0 0 + modify (ave,min,max) = 0 0 0 + total (ave,min,max) = 5.51379 5.51379 5.51379 +Step CPU Np Natt Ncoll Nreact c_c[1] c_c[2] c_c[3] c_c[4] + 0 0 99999 0 0 0 50013 30006 19980 0 + 50 0.25837697 100487 221613 15292 6 49525 30982 19980 0 + 100 0.52969439 100950 223108 15093 7 49062 31908 19980 0 + 150 0.79824022 101442 224889 15062 8 48570 32892 19980 0 + 200 1.0543701 101917 226609 15212 12 48095 33842 19980 0 + 250 1.3196561 102365 228250 15341 12 47647 34738 19980 0 + 300 1.5836764 102790 229922 15254 9 47222 35588 19980 0 + 350 1.8580471 103268 231700 15100 9 46744 36544 19980 0 + 400 2.1300416 103717 233395 15259 10 46295 37442 19980 0 +WARNING: TCE reaction probability exceeded 1.0, chemistry may be under-resolved, consider reducing timestep or fnum (further warnings suppressed) (/home/user/sparta/src/react_tce.cpp:204) + 450 2.3947466 104180 235109 15127 9 45832 38368 19980 0 + 500 2.6791988 104598 236855 15281 5 45414 39204 19980 0 +Loop time of 2.67927 on 4 procs for 500 steps with 104598 particles +Performance: 186.618 timesteps/s, 19.520 Mparticle-step/s + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Move | 0.12656 | 0.13298 | 0.13651 | 1.1 | 4.96 +Coll | 2.1289 | 2.2068 | 2.2432 | 3.1 | 82.37 +Sort | 0.048117 | 0.050791 | 0.052483 | 0.8 | 1.90 +Comm | 0.059006 | 0.061459 | 0.064185 | 0.8 | 2.29 +Modify | 0 | 0 | 0 | 0.0 | 0.00 +Output | 0.0019282 | 0.0037829 | 0.0053814 | 2.3 | 0.14 +MPI Sync| 0.1849 | 0.22321 | 0.30947 | 10.6 | 8.33 +Other | | 0.0002087 | | | 0.01 + +Particle moves = 51168099 (51.2M) +Cells touched = 56168020 (56.2M) +Particle comms = 666189 (0.666M) +Boundary collides = 0 (0K) +Boundary exits = 0 (0K) +SurfColl checks = 0 (0K) +SurfColl occurs = 0 (0K) +Surf reactions = 0 (0K) +Collide attempts = 114139243 (114M) +Collide occurs = 7614092 (7.61M) +Reactions = 4609 (4.61K) +Particles stuck = 0 +Axisymm bad moves = 0 + +Particle-moves/CPUsec/proc: 4.77445e+06 +Particle-moves/step: 102336 +Cell-touches/particle/step: 1.09772 +Particle comm iterations/step: 2.666 +Particle fraction communicated: 0.0130196 +Particle fraction colliding with boundary: 0 +Particle fraction exiting boundary: 0 +Surface-checks/particle/step: 0 +Surface-collisions/particle/step: 0 +Surf-reactions/particle/step: 0 +Collision-attempts/particle/step: 2.23067 +Collisions/particle/step: 0.148805 +Reactions/particle/step: 9.00757e-05 + +Gas reaction tallies: + style tce #-of-reactions 2 + reaction O2 + O2+ --> O + O2+ + O: 4604 + reaction O + O --> O2 + O2+: 5 + +Particles: 26149.5 ave 26219 max 26046 min +Histogram: 1 0 0 0 0 1 0 0 1 1 +Cells: 250 ave 250 max 250 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +GhostCell: 240 ave 240 max 240 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +EmptyCell: 130 ave 130 max 130 min +Histogram: 4 0 0 0 0 0 0 0 0 0 diff --git a/examples/emit/in.emit.surf.normal b/examples/emit/in.emit.surf.normal index e5f4aee73..d1a71cc5b 100644 --- a/examples/emit/in.emit.surf.normal +++ b/examples/emit/in.emit.surf.normal @@ -36,12 +36,16 @@ timestep 0.0001 # surf proc 0.01 size 512 512 zoom 1.75 #dump_modify 2 pad 4 -#compute csurf surf all all nflux mflux fx fy fz -#fix save ave/surf all 10 10 100 c_csurf[*] +# a surf tally compute driven by fix emit/surf: this is the only deck that +# exercises that dispatch path (see fix_emit_surf_kokkos.cpp) + +compute csurf surf all all nflux mflux fx fy fz +fix save ave/surf all 10 10 100 c_csurf[*] +compute cs reduce sum f_save[1] f_save[3] #dump surf surf all 100 tmp.surf id f_save[*] fix 1 balance 10 1.0 rcb part stats 100 -stats_style step cpu np nattempt ncoll nscoll nscheck +stats_style step cpu np nattempt ncoll nscoll nscheck c_cs[1] c_cs[2] run 300 diff --git a/examples/emit/log.31Oct25.mpi_1.emit.surf.normal b/examples/emit/log.22Aug26.mpi_1.emit.surf.normal similarity index 62% rename from examples/emit/log.31Oct25.mpi_1.emit.surf.normal rename to examples/emit/log.22Aug26.mpi_1.emit.surf.normal index 7b30dddd5..3668e1904 100644 --- a/examples/emit/log.31Oct25.mpi_1.emit.surf.normal +++ b/examples/emit/log.22Aug26.mpi_1.emit.surf.normal @@ -19,12 +19,12 @@ create_box 0 10 0 10 -0.5 0.5 Created orthogonal box = (0 0 -0.5) to (10 10 0.5) create_grid 10 10 1 Created 100 child grid cells - CPU time = 0.00087779 secs - create/ghost percent = 95.8557 4.14427 + CPU time = 0.000815886 secs + create/ghost percent = 94.3911 5.60887 balance_grid rcb cell Balance grid migrated 0 cells - CPU time = 9.4777e-05 secs - reassign/sort/migrate/ghost percent = 72.8331 0.274328 15.2854 11.6072 + CPU time = 0.000119338 secs + reassign/sort/migrate/ghost percent = 79.7089 0.553051 11.6551 8.08292 global nrho 1.0 fnum 0.001 @@ -41,10 +41,10 @@ read_surf data.circle 60 16 24 = cells outside/inside/overlapping surfs 24 = surf cells with 1,2,etc splits 71.8 71.8 = cell-wise and global flow volume - CPU time = 0.000600883 secs - read/check/sort/surf2grid/ghost/inout/particle percent = 14.2241 35.6459 0.415222 46.4318 3.283 13.9373 0.0166422 - surf2grid time = 0.000279001 secs - map/comm1/comm2/comm3/comm4/split percent = 35.9561 10.7297 8.19782 3.76343 20.0161 17.987 + CPU time = 0.000435757 secs + read/check/sort/surf2grid/ghost/inout/particle percent = 21.2811 27.4063 0.855293 46.3341 4.12317 7.36626 0.032128 + surf2grid time = 0.000201904 secs + map/comm1/comm2/comm3/comm4/split percent = 40.8684 11.3455 6.55906 3.6027 18.9763 16.0844 surf_collide 1 diffuse 300.0 0.0 surf_modify all collide 1 @@ -57,37 +57,44 @@ timestep 0.0001 #dump 2 image all 10 image.*.ppm type type pdiam 0.1 # surf proc 0.01 size 512 512 zoom 1.75 #dump_modify 2 pad 4 -#compute csurf surf all all nflux mflux fx fy fz -#fix save ave/surf all 10 10 100 c_csurf[*] +# a surf tally compute driven by fix emit/surf: this is the only deck that +# exercises that dispatch path (see fix_emit_surf_kokkos.cpp) + +compute csurf surf all all nflux mflux fx fy fz +fix save ave/surf all 10 10 100 c_csurf[*] +compute cs reduce sum f_save[1] f_save[3] #dump surf surf all 100 tmp.surf id f_save[*] fix 1 balance 10 1.0 rcb part stats 100 -stats_style step cpu np nattempt ncoll nscoll nscheck +stats_style step cpu np nattempt ncoll nscoll nscheck c_cs[1] c_cs[2] run 300 Memory usage per proc in Mbytes: particles (ave,min,max) = 0 0 0 grid (ave,min,max) = 1.51379 1.51379 1.51379 surf (ave,min,max) = 0.00514984 0.00514984 0.00514984 - total (ave,min,max) = 1.51894 1.51894 1.51894 -Step CPU Np Natt Ncoll Nscoll Nscheck - 0 0 0 0 0 0 0 - 100 0.01885442 6640 0 0 0 10542 - 200 0.056386249 12769 0 0 1 12750 - 300 0.10996246 17618 0 0 0 13240 -Loop time of 0.110054 on 1 procs for 300 steps with 17618 particles + modify (ave,min,max) = 0.00190735 0.00190735 0.00190735 + total (ave,min,max) = 1.52085 1.52085 1.52085 +Step CPU Np Natt Ncoll Nscoll Nscheck c_cs[1] c_cs[2] + 0 0 0 0 0 0 0 0 0 + 100 0.012855161 6640 0 0 0 10542 -1765.1285 -7.9332112e-23 + 200 0.037828025 12769 0 0 1 12750 -1765.1285 4.4239328e-23 + 300 0.075861106 17618 0 0 0 13240 -1722.6592 -1.7638632e-23 +Loop time of 0.0759187 on 1 procs for 300 steps with 17618 particles +Performance: 3951.597 timesteps/s, 69.619 Mparticle-step/s MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Move | 0.08377 | 0.08377 | 0.08377 | 0.0 | 76.12 -Coll | 0.0088308 | 0.0088308 | 0.0088308 | 0.0 | 8.02 -Sort | 0.0073681 | 0.0073681 | 0.0073681 | 0.0 | 6.69 -Comm | 0.00011004 | 0.00011004 | 0.00011004 | 0.0 | 0.10 -Modify | 0.0096948 | 0.0096948 | 0.0096948 | 0.0 | 8.81 -Output | 0.00021799 | 0.00021799 | 0.00021799 | 0.0 | 0.20 -Other | | 6.219e-05 | | | 0.06 +Move | 0.054181 | 0.054181 | 0.054181 | 0.0 | 71.37 +Coll | 0.0068936 | 0.0068936 | 0.0068936 | 0.0 | 9.08 +Sort | 0.0054101 | 0.0054101 | 0.0054101 | 0.0 | 7.13 +Comm | 0.00022189 | 0.00022189 | 0.00022189 | 0.0 | 0.29 +Modify | 0.00875 | 0.00875 | 0.00875 | 0.0 | 11.53 +Output | 0.00017001 | 0.00017001 | 0.00017001 | 0.0 | 0.22 +MPI Sync| 0.00016831 | 0.00016831 | 0.00016831 | 0.0 | 0.22 +Other | | 0.000124 | | | 0.16 Particle moves = 2854730 (2.85M) Cells touched = 2899153 (2.9M) @@ -103,7 +110,7 @@ Reactions = 0 (0K) Particles stuck = 0 Axisymm bad moves = 0 -Particle-moves/CPUsec/proc: 2.59394e+07 +Particle-moves/CPUsec/proc: 3.76025e+07 Particle-moves/step: 9515.77 Cell-touches/particle/step: 1.01556 Particle comm iterations/step: 1 diff --git a/examples/emit/log.31Oct25.mpi_4.emit.surf.normal b/examples/emit/log.22Aug26.mpi_4.emit.surf.normal similarity index 62% rename from examples/emit/log.31Oct25.mpi_4.emit.surf.normal rename to examples/emit/log.22Aug26.mpi_4.emit.surf.normal index 470106f7c..b64d0bac2 100644 --- a/examples/emit/log.31Oct25.mpi_4.emit.surf.normal +++ b/examples/emit/log.22Aug26.mpi_4.emit.surf.normal @@ -18,14 +18,14 @@ boundary o r p create_box 0 10 0 10 -0.5 0.5 Created orthogonal box = (0 0 -0.5) to (10 10 0.5) create_grid 10 10 1 -WARNING: Could not acquire nearby ghost cells b/c grid partition is not clumped (/home/runner/work/sparta/sparta/src/grid.cpp:473) +WARNING: Could not acquire nearby ghost cells b/c grid partition is not clumped (/home/user/sparta/src/grid.cpp:486) Created 100 child grid cells - CPU time = 0.00103769 secs - create/ghost percent = 94.4842 5.51581 + CPU time = 0.000976676 secs + create/ghost percent = 89.3926 10.6074 balance_grid rcb cell Balance grid migrated 74 cells - CPU time = 0.00036892 secs - reassign/sort/migrate/ghost percent = 69.4728 0.345061 12.1175 18.0646 + CPU time = 0.00047294 secs + reassign/sort/migrate/ghost percent = 59.5714 0.506618 12.1172 27.8048 global nrho 1.0 fnum 0.001 @@ -42,10 +42,10 @@ read_surf data.circle 60 16 24 = cells outside/inside/overlapping surfs 24 = surf cells with 1,2,etc splits 71.8 71.8 = cell-wise and global flow volume - CPU time = 0.000731477 secs - read/check/sort/surf2grid/ghost/inout/particle percent = 21.761 22.6268 0.350524 45.5261 9.73551 10.7518 3.34337 - surf2grid time = 0.000333013 secs - map/comm1/comm2/comm3/comm4/split percent = 28.4094 10.4095 7.68078 5.88746 18.7491 16.2822 + CPU time = 0.000750897 secs + read/check/sort/surf2grid/ghost/inout/particle percent = 10.5154 26.3955 0.483688 49.2458 13.3596 15.0543 0.164204 + surf2grid time = 0.000369785 secs + map/comm1/comm2/comm3/comm4/split percent = 27.6812 9.17371 4.86337 4.20082 33.1014 12.2842 surf_collide 1 diffuse 300.0 0.0 surf_modify all collide 1 @@ -58,37 +58,44 @@ timestep 0.0001 #dump 2 image all 10 image.*.ppm type type pdiam 0.1 # surf proc 0.01 size 512 512 zoom 1.75 #dump_modify 2 pad 4 -#compute csurf surf all all nflux mflux fx fy fz -#fix save ave/surf all 10 10 100 c_csurf[*] +# a surf tally compute driven by fix emit/surf: this is the only deck that +# exercises that dispatch path (see fix_emit_surf_kokkos.cpp) + +compute csurf surf all all nflux mflux fx fy fz +fix save ave/surf all 10 10 100 c_csurf[*] +compute cs reduce sum f_save[1] f_save[3] #dump surf surf all 100 tmp.surf id f_save[*] fix 1 balance 10 1.0 rcb part stats 100 -stats_style step cpu np nattempt ncoll nscoll nscheck +stats_style step cpu np nattempt ncoll nscoll nscheck c_cs[1] c_cs[2] run 300 Memory usage per proc in Mbytes: particles (ave,min,max) = 0 0 0 grid (ave,min,max) = 1.51379 1.51379 1.51379 surf (ave,min,max) = 0.00514984 0.00514984 0.00514984 - total (ave,min,max) = 1.51894 1.51894 1.51894 -Step CPU Np Natt Ncoll Nscoll Nscheck - 0 0 0 0 0 0 0 - 100 0.009278268 6630 0 0 0 10740 - 200 0.022921819 12701 0 0 0 12442 - 300 0.040733984 17485 0 0 8 13474 -Loop time of 0.0407694 on 4 procs for 300 steps with 17485 particles + modify (ave,min,max) = 0.000476837 0.000457764 0.000495911 + total (ave,min,max) = 1.51942 1.5194 1.51944 +Step CPU Np Natt Ncoll Nscoll Nscheck c_cs[1] c_cs[2] + 0 0 0 0 0 0 0 0 0 + 100 0.007255719 6630 0 0 0 10740 -1796.9804 9.1920834e-24 + 200 0.015957201 12701 0 0 0 12442 -1741.2395 -2.9206075e-24 + 300 0.026809096 17485 0 0 8 13474 -1762.4741 -1.1401681e-23 +Loop time of 0.0268685 on 4 procs for 300 steps with 17485 particles +Performance: 11165.483 timesteps/s, 195.228 Mparticle-step/s MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Move | 0.021798 | 0.021987 | 0.022133 | 0.1 | 53.93 -Coll | 0.0016594 | 0.0016718 | 0.0017 | 0.0 | 4.10 -Sort | 0.0036347 | 0.0036888 | 0.0037699 | 0.1 | 9.05 -Comm | 0.002952 | 0.0030522 | 0.0031856 | 0.2 | 7.49 -Modify | 0.0082378 | 0.0082735 | 0.0083231 | 0.0 | 20.29 -Output | 3.9053e-05 | 6.4311e-05 | 0.00013803 | 0.0 | 0.16 -Other | | 0.002032 | | | 4.98 +Move | 0.010738 | 0.011997 | 0.013201 | 0.8 | 44.65 +Coll | 0.0013471 | 0.0014589 | 0.0015055 | 0.2 | 5.43 +Sort | 0.00077429 | 0.0008663 | 0.00096122 | 0.0 | 3.22 +Comm | 0.0023224 | 0.0023519 | 0.0023753 | 0.0 | 8.75 +Modify | 0.0065119 | 0.0066031 | 0.0066903 | 0.1 | 24.58 +Output | 4.5875e-05 | 8.68e-05 | 0.00020941 | 0.0 | 0.32 +MPI Sync| 0.0020302 | 0.0034611 | 0.0049766 | 1.8 | 12.88 +Other | | 4.337e-05 | | | 0.16 Particle moves = 2834836 (2.83M) Cells touched = 2881345 (2.88M) @@ -104,7 +111,7 @@ Reactions = 0 (0K) Particles stuck = 0 Axisymm bad moves = 0 -Particle-moves/CPUsec/proc: 1.73833e+07 +Particle-moves/CPUsec/proc: 2.63769e+07 Particle-moves/step: 9449.45 Cell-touches/particle/step: 1.01641 Particle comm iterations/step: 1.93 diff --git a/examples/flowfile/in.flowfile b/examples/flowfile/in.flowfile index abc76de2e..a869d9f62 100644 --- a/examples/flowfile/in.flowfile +++ b/examples/flowfile/in.flowfile @@ -22,7 +22,7 @@ global nrho 1.0 fnum 0.001 species air.species N O mixture air N O vstream 100.0 0 0 -fix in emit/face/file air xlo flow.face XLO frac 0.5 +fix in emit/face/file air xlo flow.face XLO frac 0.5 twopass timestep 0.0001 diff --git a/examples/flowfile/log.11Sep23.mpi_4.flowfile b/examples/flowfile/log.11Sep23.mpi_4.flowfile deleted file mode 100644 index 82214d507..000000000 --- a/examples/flowfile/log.11Sep23.mpi_4.flowfile +++ /dev/null @@ -1,113 +0,0 @@ -SPARTA (13 Apr 2023) -Running on 4 MPI task(s) -################################################################################ -# 2d flow profile input from file -# -# Note: -# - The "comm/sort” option to the “global” command is used to match MPI runs. -# - The “twopass” option is used to match Kokkos runs. -# The "comm/sort" and "twopass" options should not be used for production runs. -################################################################################ - -seed 12345 -dimension 2 -global gridcut 0.0 comm/sort yes - -boundary o r p - -create_box 0 10 0 10 -0.5 0.5 -Created orthogonal box = (0 0 -0.5) to (10 10 0.5) -create_grid 20 20 1 -WARNING: Could not acquire nearby ghost cells b/c grid partition is not clumped (/home/runner/work/sparta/sparta/src/grid.cpp:465) -Created 400 child grid cells - CPU time = 0.0019634 secs - create/ghost percent = 90.547 9.45298 -balance_grid rcb cell -Balance grid migrated 280 cells - CPU time = 0.001039 secs - reassign/sort/migrate/ghost percent = 45.9096 0.567853 22.3099 31.2127 - -global nrho 1.0 fnum 0.001 - -species air.species N O -mixture air N O vstream 100.0 0 0 - -fix in emit/face/file air xlo flow.face XLO frac 0.5 - -timestep 0.0001 - -#dump 2 image all 50 image.*.ppm type type pdiam 0.1 # surf proc 0.01 size 512 512 zoom 1.75 -#dump_modify 2 pad 4 - -fix 1 balance 100 1.0 rcb part - -stats 100 -stats_style step cpu np nattempt ncoll nscoll nscheck f_1 f_1[1] f_1[2] - -run 1000 -Memory usage per proc in Mbytes: - particles (ave,min,max) = 0 0 0 - grid (ave,min,max) = 1.51379 1.51379 1.51379 - surf (ave,min,max) = 0 0 0 - total (ave,min,max) = 1.51379 1.51379 1.51379 -Step CPU Np Natt Ncoll Nscoll Nscheck f_1 f_1[1] f_1[2] - 0 0 0 0 0 0 0 1 0 1 - 100 0.010664109 6008 0 0 0 0 1.0213049 1534 2.0013316 - 200 0.02266262 11969 0 0 0 0 1.009274 3020 1.4998747 - 300 0.037309833 17932 0 0 0 0 1.0129378 4541 1.3455275 - 400 0.055407249 22989 0 0 0 0 1.0105703 5808 1.2270216 - 500 0.075395566 25992 0 0 0 0 1.0040012 6524 1.1131117 - 600 0.097083785 27909 0 0 0 0 1.0099968 7047 1.066896 - 700 0.12008031 28952 0 0 0 0 1.0084277 7299 1.0414479 - 800 0.14263263 29654 0 0 0 0 1.0089701 7480 1.0252917 - 900 0.16577755 29975 0 0 0 0 1.0028357 7515 1.017648 - 1000 0.18864367 30042 0 0 0 0 1.0072565 7565 1.0097863 -Loop time of 0.188259 on 4 procs for 1000 steps with 30042 particles - -MPI task timing breakdown: -Section | min time | avg time | max time |%varavg| %total ---------------------------------------------------------------- -Move | 0.048203 | 0.052486 | 0.056841 | 1.3 | 27.88 -Coll | 0 | 0 | 0 | 0.0 | 0.00 -Sort | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.049808 | 0.069597 | 0.089243 | 5.4 | 36.97 -Modify | 0.010013 | 0.014041 | 0.018379 | 3.3 | 7.46 -Output | 0.0005328 | 0.00089615 | 0.0010869 | 0.0 | 0.48 -Other | | 0.05124 | | | 27.22 - -Particle moves = 21696764 (21.7M) -Cells touched = 22664759 (22.7M) -Particle comms = 75451 (75.5K) -Boundary collides = 102 (0.102K) -Boundary exits = 29988 (30K) -SurfColl checks = 0 (0K) -SurfColl occurs = 0 (0K) -Surf reactions = 0 (0K) -Collide attempts = 0 (0K) -Collide occurs = 0 (0K) -Reactions = 0 (0K) -Particles stuck = 0 -Axisymm bad moves = 0 - -Particle-moves/CPUsec/proc: 2.88124e+07 -Particle-moves/step: 21696.8 -Cell-touches/particle/step: 1.04461 -Particle comm iterations/step: 1.992 -Particle fraction communicated: 0.00347752 -Particle fraction colliding with boundary: 4.70116e-06 -Particle fraction exiting boundary: 0.00138214 -Surface-checks/particle/step: 0 -Surface-collisions/particle/step: 0 -Surf-reactions/particle/step: 0 -Collision-attempts/particle/step: 0 -Collisions/particle/step: 0 -Reactions/particle/step: 0 - -Particles: 7510.5 ave 7565 max 7467 min -Histogram: 1 0 0 1 1 0 0 0 0 1 -Cells: 100 ave 103 max 98 min -Histogram: 1 0 1 0 1 0 0 0 0 1 -GhostCell: 26.5 ave 34 max 21 min -Histogram: 2 0 0 0 0 0 1 0 0 1 -EmptyCell: 21.5 ave 22 max 21 min -Histogram: 2 0 0 0 0 0 0 0 0 2 diff --git a/examples/flowfile/log.11Sep23.mpi_1.flowfile b/examples/flowfile/log.22Aug26.mpi_1.flowfile similarity index 54% rename from examples/flowfile/log.11Sep23.mpi_1.flowfile rename to examples/flowfile/log.22Aug26.mpi_1.flowfile index e083688a3..eb34b5b80 100644 --- a/examples/flowfile/log.11Sep23.mpi_1.flowfile +++ b/examples/flowfile/log.22Aug26.mpi_1.flowfile @@ -1,4 +1,4 @@ -SPARTA (13 Apr 2023) +SPARTA (24 Sep 2025) Running on 1 MPI task(s) ################################################################################ # 2d flow profile input from file @@ -19,19 +19,19 @@ create_box 0 10 0 10 -0.5 0.5 Created orthogonal box = (0 0 -0.5) to (10 10 0.5) create_grid 20 20 1 Created 400 child grid cells - CPU time = 0.000922203 secs - create/ghost percent = 91.2925 8.70752 + CPU time = 0.00127087 secs + create/ghost percent = 89.7074 10.2926 balance_grid rcb cell Balance grid migrated 0 cells - CPU time = 0.0001343 secs - reassign/sort/migrate/ghost percent = 61.2807 0.595681 12.2115 25.9121 + CPU time = 0.000182925 secs + reassign/sort/migrate/ghost percent = 56.0399 0.766981 9.75372 33.4394 global nrho 1.0 fnum 0.001 species air.species N O mixture air N O vstream 100.0 0 0 -fix in emit/face/file air xlo flow.face XLO frac 0.5 +fix in emit/face/file air xlo flow.face XLO frac 0.5 twopass timestep 0.0001 @@ -48,37 +48,40 @@ Memory usage per proc in Mbytes: particles (ave,min,max) = 0 0 0 grid (ave,min,max) = 1.51379 1.51379 1.51379 surf (ave,min,max) = 0 0 0 + modify (ave,min,max) = 0 0 0 total (ave,min,max) = 1.51379 1.51379 1.51379 Step CPU Np Natt Ncoll Nscoll Nscheck f_1 f_1[1] f_1[2] 0 0 0 0 0 0 0 1 0 1 - 100 0.006801015 6015 0 0 0 0 1 6015 1 - 200 0.027320663 12027 0 0 0 0 1 12027 1 - 300 0.059719937 18003 0 0 0 0 1 18003 1 - 400 0.10448904 23027 0 0 0 0 1 23027 1 - 500 0.15345095 26084 0 0 0 0 1 26084 1 - 600 0.20271637 27868 0 0 0 0 1 27868 1 - 700 0.2631821 28943 0 0 0 0 1 28943 1 - 800 0.31603033 29588 0 0 0 0 1 29588 1 - 900 0.37706837 29849 0 0 0 0 1 29849 1 - 1000 0.4370831 29946 0 0 0 0 1 29946 1 -Loop time of 0.44386 on 1 procs for 1000 steps with 29946 particles + 100 0.005448947 6015 0 0 0 0 1 6015 1 + 200 0.016242625 12020 0 0 0 0 1 12020 1 + 300 0.034978374 18023 0 0 0 0 1 18023 1 + 400 0.05786652 23088 0 0 0 0 1 23088 1 + 500 0.084913334 26137 0 0 0 0 1 26137 1 + 600 0.11385508 27886 0 0 0 0 1 27886 1 + 700 0.14476831 29000 0 0 0 0 1 29000 1 + 800 0.17656475 29595 0 0 0 0 1 29595 1 + 900 0.2085744 29918 0 0 0 0 1 29918 1 + 1000 0.24162434 30027 0 0 0 0 1 30027 1 +Loop time of 0.241684 on 1 procs for 1000 steps with 30027 particles +Performance: 4137.636 timesteps/s, 124.241 Mparticle-step/s MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Move | 0.38913 | 0.38913 | 0.38913 | 0.0 | 87.67 +Move | 0.22187 | 0.22187 | 0.22187 | 0.0 | 91.80 Coll | 0 | 0 | 0 | 0.0 | 0.00 Sort | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.0006528 | 0.0006528 | 0.0006528 | 0.0 | 0.15 -Modify | 0.016625 | 0.016625 | 0.016625 | 0.0 | 3.75 -Output | 0.037214 | 0.037214 | 0.037214 | 0.0 | 8.38 -Other | | 0.0002427 | | | 0.05 +Comm | 0.001179 | 0.001179 | 0.001179 | 0.0 | 0.49 +Modify | 0.01726 | 0.01726 | 0.01726 | 0.0 | 7.14 +Output | 0.00060475 | 0.00060475 | 0.00060475 | 0.0 | 0.25 +MPI Sync| 0.00069279 | 0.00069279 | 0.00069279 | 0.0 | 0.29 +Other | | 7.256e-05 | | | 0.03 -Particle moves = 21737175 (21.7M) -Cells touched = 22674296 (22.7M) +Particle moves = 21762921 (21.8M) +Cells touched = 22702135 (22.7M) Particle comms = 0 (0K) -Boundary collides = 99 (0.099K) -Boundary exits = 30032 (30K) +Boundary collides = 96 (0.096K) +Boundary exits = 30033 (30K) SurfColl checks = 0 (0K) SurfColl occurs = 0 (0K) Surf reactions = 0 (0K) @@ -88,13 +91,13 @@ Reactions = 0 (0K) Particles stuck = 0 Axisymm bad moves = 0 -Particle-moves/CPUsec/proc: 4.8973e+07 -Particle-moves/step: 21737.2 -Cell-touches/particle/step: 1.04311 +Particle-moves/CPUsec/proc: 9.0047e+07 +Particle-moves/step: 21762.9 +Cell-touches/particle/step: 1.04316 Particle comm iterations/step: 1 Particle fraction communicated: 0 -Particle fraction colliding with boundary: 4.55441e-06 -Particle fraction exiting boundary: 0.0013816 +Particle fraction colliding with boundary: 4.41117e-06 +Particle fraction exiting boundary: 0.00138001 Surface-checks/particle/step: 0 Surface-collisions/particle/step: 0 Surf-reactions/particle/step: 0 @@ -102,7 +105,7 @@ Collision-attempts/particle/step: 0 Collisions/particle/step: 0 Reactions/particle/step: 0 -Particles: 29946 ave 29946 max 29946 min +Particles: 30027 ave 30027 max 30027 min Histogram: 1 0 0 0 0 0 0 0 0 0 Cells: 400 ave 400 max 400 min Histogram: 1 0 0 0 0 0 0 0 0 0 diff --git a/examples/flowfile/log.22Aug26.mpi_4.flowfile b/examples/flowfile/log.22Aug26.mpi_4.flowfile new file mode 100644 index 000000000..bfeb721c6 --- /dev/null +++ b/examples/flowfile/log.22Aug26.mpi_4.flowfile @@ -0,0 +1,116 @@ +SPARTA (24 Sep 2025) +Running on 4 MPI task(s) +################################################################################ +# 2d flow profile input from file +# +# Note: +# - The "comm/sort” option to the “global” command is used to match MPI runs. +# - The “twopass” option is used to match Kokkos runs. +# The "comm/sort" and "twopass" options should not be used for production runs. +################################################################################ + +seed 12345 +dimension 2 +global gridcut 0.0 comm/sort yes + +boundary o r p + +create_box 0 10 0 10 -0.5 0.5 +Created orthogonal box = (0 0 -0.5) to (10 10 0.5) +create_grid 20 20 1 +WARNING: Could not acquire nearby ghost cells b/c grid partition is not clumped (/home/user/sparta/src/grid.cpp:486) +Created 400 child grid cells + CPU time = 0.00128677 secs + create/ghost percent = 95.2517 4.74833 +balance_grid rcb cell +Balance grid migrated 280 cells + CPU time = 0.000546427 secs + reassign/sort/migrate/ghost percent = 52.9452 0.419086 24.9869 21.6488 + +global nrho 1.0 fnum 0.001 + +species air.species N O +mixture air N O vstream 100.0 0 0 + +fix in emit/face/file air xlo flow.face XLO frac 0.5 twopass + +timestep 0.0001 + +#dump 2 image all 50 image.*.ppm type type pdiam 0.1 # surf proc 0.01 size 512 512 zoom 1.75 +#dump_modify 2 pad 4 + +fix 1 balance 100 1.0 rcb part + +stats 100 +stats_style step cpu np nattempt ncoll nscoll nscheck f_1 f_1[1] f_1[2] + +run 1000 +Memory usage per proc in Mbytes: + particles (ave,min,max) = 0 0 0 + grid (ave,min,max) = 1.51379 1.51379 1.51379 + surf (ave,min,max) = 0 0 0 + modify (ave,min,max) = 0 0 0 + total (ave,min,max) = 1.51379 1.51379 1.51379 +Step CPU Np Natt Ncoll Nscoll Nscheck f_1 f_1[1] f_1[2] + 0 0 0 0 0 0 0 1 0 1 + 100 0.007064918 5966 0 0 0 0 1.034529 1543 2.020114 + 200 0.01233207 11950 0 0 0 0 1.0132218 3027 1.4948954 + 300 0.019997798 17937 0 0 0 0 1.0077493 4519 1.3957741 + 400 0.028136009 22996 0 0 0 0 1.0114803 5815 1.2245608 + 500 0.036668081 26046 0 0 0 0 1.0049912 6544 1.1230899 + 600 0.0464292 27840 0 0 0 0 1.0135057 7054 1.0666667 + 700 0.056076516 28950 0 0 0 0 1.0031088 7260 1.0453886 + 800 0.066188672 29587 0 0 0 0 1.0062527 7443 1.0197722 + 900 0.077706447 29851 0 0 0 0 1.0032495 7487 1.0208033 + 1000 0.089577212 29996 0 0 0 0 1.0082678 7561 1.0082678 +Loop time of 0.0896553 on 4 procs for 1000 steps with 29996 particles +Performance: 11153.833 timesteps/s, 334.570 Mparticle-step/s + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Move | 0.04725 | 0.051253 | 0.054016 | 1.1 | 57.17 +Coll | 0 | 0 | 0 | 0.0 | 0.00 +Sort | 0 | 0 | 0 | 0.0 | 0.00 +Comm | 0.012415 | 0.012752 | 0.013283 | 0.3 | 14.22 +Modify | 0.0076533 | 0.011332 | 0.0151 | 3.5 | 12.64 +Output | 0.00013441 | 0.000259 | 0.00062495 | 0.0 | 0.29 +MPI Sync| 0.010715 | 0.014018 | 0.015781 | 1.7 | 15.64 +Other | | 4.151e-05 | | | 0.05 + +Particle moves = 21673582 (21.7M) +Cells touched = 22639675 (22.6M) +Particle comms = 76481 (76.5K) +Boundary collides = 89 (0.089K) +Boundary exits = 29908 (29.9K) +SurfColl checks = 0 (0K) +SurfColl occurs = 0 (0K) +Surf reactions = 0 (0K) +Collide attempts = 0 (0K) +Collide occurs = 0 (0K) +Reactions = 0 (0K) +Particles stuck = 0 +Axisymm bad moves = 0 + +Particle-moves/CPUsec/proc: 6.04359e+07 +Particle-moves/step: 21673.6 +Cell-touches/particle/step: 1.04457 +Particle comm iterations/step: 1.991 +Particle fraction communicated: 0.00352877 +Particle fraction colliding with boundary: 4.10638e-06 +Particle fraction exiting boundary: 0.00137993 +Surface-checks/particle/step: 0 +Surface-collisions/particle/step: 0 +Surf-reactions/particle/step: 0 +Collision-attempts/particle/step: 0 +Collisions/particle/step: 0 +Reactions/particle/step: 0 + +Particles: 7499 ave 7561 max 7456 min +Histogram: 2 0 0 0 0 1 0 0 0 1 +Cells: 100 ave 100 max 100 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +GhostCell: 21 ave 21 max 21 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +EmptyCell: 21 ave 21 max 21 min +Histogram: 4 0 0 0 0 0 0 0 0 0 diff --git a/examples/free/in.free b/examples/free/in.free index 348ff08a8..cda55963f 100644 --- a/examples/free/in.free +++ b/examples/free/in.free @@ -29,7 +29,19 @@ create_particles air n 10000 twopass stats 100 compute temp temp -stats_style step cpu np nattempt ncoll c_temp + +# compute boundary drives the boundary-tally path in Update; every box face +# here is reflecting, so all six tally. compute ke/particle is reduced to a +# scalar so it can appear in stats. + +# every box face here is specular (rr), which conserves energy and applies no +# shear, so ke/erot/evib and shx/shy/shz would be identically zero and could +# not detect a regression. n/nwt/press are the values that actually vary. + +compute bound boundary all n nwt press + +stats_style step cpu np nattempt ncoll c_temp & + c_bound[1][1] c_bound[1][3] c_bound[4][3] c_bound[6][3] #dump 2 image all 100 image.*.ppm type type pdiam 3.0e-6 & # size 512 512 gline yes 0.005 diff --git a/examples/free/in.free.restart b/examples/free/in.free.restart new file mode 100644 index 000000000..cf1b05f54 --- /dev/null +++ b/examples/free/in.free.restart @@ -0,0 +1,65 @@ +################################################################################ +# restart round trip: write a restart file, read it back, continue the run +# +# This is the only deck that exercises read_restart in CI -- the four +# examples/custom/in.custom.*.restart decks are all in SPARTA_DISABLED_TESTS. +# +# All three filename-substitution paths are driven deliberately: +# restart -> Output::write() (periodic restart output) +# write_restart -> WriteRestart::command() +# read_restart -> ReadRestart::file_search() (the "*" wildcard search) +# Each replaces "*" with a timestep, and each sizes the substituted name +# itself. The ".equil.restart" suffix is deliberately long: a name is +# truncated only when (timestep digits + suffix length) exceeds 15, so a +# short suffix would let a sizing bug through unnoticed. +# +# The file name carries the MPI rank count, because ctest runs the mpi_1 and +# mpi_4 copies of a test concurrently in one shared suite directory and they +# would otherwise write the same file (compare the note on +# in.exp2imp.axi.spherecone.readback in SPARTA_DISABLED_TESTS). +# +# Note: +# - The "comm/sort" option to the "global" command is used to match MPI runs. +# - The "twopass" option is used to match Kokkos runs. +# The "comm/sort" and "twopass" options should not be used for production runs. +################################################################################ + +seed 12345 +dimension 3 +global gridcut 1.0e-5 comm/sort yes + +boundary rr rr rr + +create_box 0 0.0001 0 0.0001 0 0.0001 +create_grid 4 4 4 +balance_grid rcb part + +species ar.species Ar +mixture air Ar vstream 0.0 0.0 0.0 temp 273.15 + +global nrho 7.07043E22 fnum 7.07043E7 + +create_particles air n 2000 twopass + +compute temp temp +stats 10 +stats_style step np c_temp + +timestep 7.00E-9 + +variable np equal extract_setting(world_size) +restart 10 tmp.free.periodic.np${np}.*.equil.restart +run 20 +write_restart tmp.free.np${np}.*.equil.restart + +# read the restart back through the "*" wildcard search and continue + +clear +seed 12345 +variable np equal extract_setting(world_size) +read_restart tmp.free.np${np}.*.equil.restart + +compute temp temp +stats 10 +stats_style step np c_temp +run 20 diff --git a/examples/free/log.11Sep23.mpi_4.free b/examples/free/log.11Sep23.mpi_4.free deleted file mode 100644 index 99e973500..000000000 --- a/examples/free/log.11Sep23.mpi_4.free +++ /dev/null @@ -1,116 +0,0 @@ -SPARTA (13 Apr 2023) -Running on 4 MPI task(s) -################################################################################ -# thermal gas in a 3d box with free molecular flow (no collisions) -# -# Note: -# - The "comm/sort” option to the “global” command is used to match MPI runs. -# - The “twopass” option is used to match Kokkos runs. -# The "comm/sort" and "twopass" options should not be used for production runs. -################################################################################ -# particles reflect off global box boundaries - -seed 12345 -dimension 3 -global gridcut 1.0e-5 comm/sort yes - -boundary rr rr rr - -create_box 0 0.0001 0 0.0001 0 0.0001 -Created orthogonal box = (0 0 0) to (0.0001 0.0001 0.0001) -create_grid 10 10 10 -WARNING: Could not acquire nearby ghost cells b/c grid partition is not clumped (/home/runner/work/sparta/sparta/src/grid.cpp:465) -Created 1000 child grid cells - CPU time = 0.0018284 secs - create/ghost percent = 92.8571 7.14285 - -balance_grid rcb part -Balance grid migrated 740 cells - CPU time = 0.0012836 secs - reassign/sort/migrate/ghost percent = 45.6762 0.498597 18.4948 35.3303 - -species ar.species Ar -mixture air Ar vstream 0.0 0.0 0.0 temp 273.15 - -global nrho 7.07043E22 -global fnum 7.07043E6 - -create_particles air n 10000 twopass -Created 10000 particles - CPU time = 0.0026624 secs - -stats 100 -compute temp temp -stats_style step cpu np nattempt ncoll c_temp - -#dump 2 image all 100 image.*.ppm type type pdiam 3.0e-6 # size 512 512 gline yes 0.005 -#dump_modify 2 pad 4 - -timestep 7.00E-9 -run 1000 -Memory usage per proc in Mbytes: - particles (ave,min,max) = 1.6875 1.6875 1.6875 - grid (ave,min,max) = 1.51379 1.51379 1.51379 - surf (ave,min,max) = 0 0 0 - total (ave,min,max) = 3.20129 3.20129 3.20129 -Step CPU Np Natt Ncoll c_temp - 0 0 10000 0 0 274.13466 - 100 0.016824415 10000 0 0 274.13466 - 200 0.03390443 10000 0 0 274.13466 - 300 0.051135146 10000 0 0 274.13466 - 400 0.068309761 10000 0 0 274.13466 - 500 0.085753376 10000 0 0 274.13466 - 600 0.10248639 10000 0 0 274.13466 - 700 0.11956671 10000 0 0 274.13466 - 800 0.13623952 10000 0 0 274.13466 - 900 0.15316264 10000 0 0 274.13466 - 1000 0.17418935 10000 0 0 274.13466 -Loop time of 0.174222 on 4 procs for 1000 steps with 10000 particles - -MPI task timing breakdown: -Section | min time | avg time | max time |%varavg| %total ---------------------------------------------------------------- -Move | 0.055368 | 0.055797 | 0.056042 | 0.1 | 32.03 -Coll | 0 | 0 | 0 | 0.0 | 0.00 -Sort | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.028104 | 0.06845 | 0.092353 | 9.3 | 39.29 -Modify | 0 | 0 | 0 | 0.0 | 0.00 -Output | 0.0005084 | 0.00085532 | 0.0012147 | 0.0 | 0.49 -Other | | 0.04912 | | | 28.19 - -Particle moves = 10000000 (10M) -Cells touched = 13601534 (13.6M) -Particle comms = 264033 (0.264M) -Boundary collides = 400226 (0.4M) -Boundary exits = 0 (0K) -SurfColl checks = 0 (0K) -SurfColl occurs = 0 (0K) -Surf reactions = 0 (0K) -Collide attempts = 0 (0K) -Collide occurs = 0 (0K) -Reactions = 0 (0K) -Particles stuck = 0 -Axisymm bad moves = 0 - -Particle-moves/CPUsec/proc: 1.43495e+07 -Particle-moves/step: 10000 -Cell-touches/particle/step: 1.36015 -Particle comm iterations/step: 1 -Particle fraction communicated: 0.0264033 -Particle fraction colliding with boundary: 0.0400226 -Particle fraction exiting boundary: 0 -Surface-checks/particle/step: 0 -Surface-collisions/particle/step: 0 -Surf-reactions/particle/step: 0 -Collision-attempts/particle/step: 0 -Collisions/particle/step: 0 -Reactions/particle/step: 0 - -Particles: 2500 ave 2571 max 2423 min -Histogram: 1 0 0 0 0 2 0 0 0 1 -Cells: 250 ave 250 max 250 min -Histogram: 4 0 0 0 0 0 0 0 0 0 -GhostCell: 172.5 ave 240 max 110 min -Histogram: 1 0 0 0 2 0 0 0 0 1 -EmptyCell: 62.5 ave 130 max 0 min -Histogram: 1 0 0 0 2 0 0 0 0 1 diff --git a/examples/free/log.11Sep23.mpi_1.free b/examples/free/log.22Aug26.mpi_1.free similarity index 50% rename from examples/free/log.11Sep23.mpi_1.free rename to examples/free/log.22Aug26.mpi_1.free index 18114fe97..65f403748 100644 --- a/examples/free/log.11Sep23.mpi_1.free +++ b/examples/free/log.22Aug26.mpi_1.free @@ -1,4 +1,4 @@ -SPARTA (13 Apr 2023) +SPARTA (24 Sep 2025) Running on 1 MPI task(s) ################################################################################ # thermal gas in a 3d box with free molecular flow (no collisions) @@ -20,13 +20,13 @@ create_box 0 0.0001 0 0.0001 0 0.0001 Created orthogonal box = (0 0 0) to (0.0001 0.0001 0.0001) create_grid 10 10 10 Created 1000 child grid cells - CPU time = 0.0010424 secs - create/ghost percent = 79.7102 20.2898 + CPU time = 0.00111567 secs + create/ghost percent = 82.2533 17.7467 balance_grid rcb part Balance grid migrated 0 cells - CPU time = 0.0002467 secs - reassign/sort/migrate/ghost percent = 53.182 0.486421 10.4986 35.833 + CPU time = 0.000233845 secs + reassign/sort/migrate/ghost percent = 59.241 0.441746 8.95593 31.3614 species ar.species Ar mixture air Ar vstream 0.0 0.0 0.0 temp 273.15 @@ -36,11 +36,22 @@ global fnum 7.07043E6 create_particles air n 10000 twopass Created 10000 particles - CPU time = 0.00231581 secs + CPU time = 0.00188917 secs stats 100 compute temp temp -stats_style step cpu np nattempt ncoll c_temp + +# compute boundary drives the boundary-tally path in Update; every box face +# here is reflecting, so all six tally. compute ke/particle is reduced to a +# scalar so it can appear in stats. + +# every box face here is specular (rr), which conserves energy and applies no +# shear, so ke/erot/evib and shx/shy/shz would be identically zero and could +# not detect a regression. n/nwt/press are the values that actually vary. + +compute bound boundary all n nwt press + +stats_style step cpu np nattempt ncoll c_temp c_bound[1][1] c_bound[1][3] c_bound[4][3] c_bound[6][3] #dump 2 image all 100 image.*.ppm type type pdiam 3.0e-6 # size 512 512 gline yes 0.005 #dump_modify 2 pad 4 @@ -48,34 +59,37 @@ stats_style step cpu np nattempt ncoll c_temp timestep 7.00E-9 run 1000 Memory usage per proc in Mbytes: - particles (ave,min,max) = 1.6875 1.6875 1.6875 + particles (ave,min,max) = 1.5625 1.5625 1.5625 grid (ave,min,max) = 1.51379 1.51379 1.51379 surf (ave,min,max) = 0 0 0 - total (ave,min,max) = 3.20129 3.20129 3.20129 -Step CPU Np Natt Ncoll c_temp - 0 0 10000 0 0 273.86304 - 100 0.054227225 10000 0 0 273.86304 - 200 0.10290524 10000 0 0 273.86304 - 300 0.15085215 10000 0 0 273.86304 - 400 0.19381234 10000 0 0 273.86304 - 500 0.23473094 10000 0 0 273.86304 - 600 0.28685016 10000 0 0 273.86304 - 700 0.33477357 10000 0 0 273.86304 - 800 0.37378336 10000 0 0 273.86304 - 900 0.41886306 10000 0 0 273.86304 - 1000 0.47083758 10000 0 0 273.86304 -Loop time of 0.470863 on 1 procs for 1000 steps with 10000 particles + modify (ave,min,max) = 0 0 0 + total (ave,min,max) = 3.07629 3.07629 3.07629 +Step CPU Np Natt Ncoll c_temp c_bound[1][1] c_bound[1][3] c_bound[4][3] c_bound[6][3] + 0 0 10000 0 0 273.86304 0 0 0 0 + 100 0.016827341 10000 0 0 273.86304 57 214.24854 251.74501 271.1099 + 200 0.032279651 10000 0 0 273.86304 63 255.90951 348.55827 219.21443 + 300 0.047780964 10000 0 0 273.86304 78 295.95887 337.79612 276.83177 + 400 0.063203253 10000 0 0 273.86304 68 292.68625 219.51827 207.31046 + 500 0.078529368 10000 0 0 273.86304 61 241.63957 258.29734 307.34431 + 600 0.094091331 10000 0 0 273.86304 74 334.64697 265.13811 310.94753 + 700 0.10976898 10000 0 0 273.86304 76 334.09186 314.4322 265.22728 + 800 0.12525419 10000 0 0 273.86304 68 264.64787 237.79326 310.70583 + 900 0.14098636 10000 0 0 273.86304 75 288.00026 289.11476 189.79319 + 1000 0.15667958 10000 0 0 273.86304 69 273.93655 258.0403 287.43626 +Loop time of 0.156709 on 1 procs for 1000 steps with 10000 particles +Performance: 6381.261 timesteps/s, 63.813 Mparticle-step/s MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Move | 0.4282 | 0.4282 | 0.4282 | 0.0 | 90.94 +Move | 0.15552 | 0.15552 | 0.15552 | 0.0 | 99.24 Coll | 0 | 0 | 0 | 0.0 | 0.00 Sort | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.0001422 | 0.0001422 | 0.0001422 | 0.0 | 0.03 +Comm | 0.00030859 | 0.00030859 | 0.00030859 | 0.0 | 0.20 Modify | 0 | 0 | 0 | 0.0 | 0.00 -Output | 0.04235 | 0.04235 | 0.04235 | 0.0 | 8.99 -Other | | 0.000175 | | | 0.04 +Output | 0.00048098 | 0.00048098 | 0.00048098 | 0.0 | 0.31 +MPI Sync| 0.0002613 | 0.0002613 | 0.0002613 | 0.0 | 0.17 +Other | | 0.0001414 | | | 0.09 Particle moves = 10000000 (10M) Cells touched = 13599661 (13.6M) @@ -91,7 +105,7 @@ Reactions = 0 (0K) Particles stuck = 0 Axisymm bad moves = 0 -Particle-moves/CPUsec/proc: 2.12376e+07 +Particle-moves/CPUsec/proc: 6.38126e+07 Particle-moves/step: 10000 Cell-touches/particle/step: 1.35997 Particle comm iterations/step: 1 diff --git a/examples/free/log.22Aug26.mpi_1.free.restart b/examples/free/log.22Aug26.mpi_1.free.restart new file mode 100644 index 000000000..2f5441eed --- /dev/null +++ b/examples/free/log.22Aug26.mpi_1.free.restart @@ -0,0 +1,207 @@ +SPARTA (24 Sep 2025) +Running on 1 MPI task(s) +################################################################################ +# restart round trip: write a restart file, read it back, continue the run +# +# This is the only deck that exercises read_restart in CI -- the four +# examples/custom/in.custom.*.restart decks are all in SPARTA_DISABLED_TESTS. +# +# All three filename-substitution paths are driven deliberately: +# restart -> Output::write() (periodic restart output) +# write_restart -> WriteRestart::command() +# read_restart -> ReadRestart::file_search() (the "*" wildcard search) +# Each replaces "*" with a timestep, and each sizes the substituted name +# itself. The ".equil.restart" suffix is deliberately long: a name is +# truncated only when (timestep digits + suffix length) exceeds 15, so a +# short suffix would let a sizing bug through unnoticed. +# +# The file name carries the MPI rank count, because ctest runs the mpi_1 and +# mpi_4 copies of a test concurrently in one shared suite directory and they +# would otherwise write the same file (compare the note on +# in.exp2imp.axi.spherecone.readback in SPARTA_DISABLED_TESTS). +# +# Note: +# - The "comm/sort" option to the "global" command is used to match MPI runs. +# - The "twopass" option is used to match Kokkos runs. +# The "comm/sort" and "twopass" options should not be used for production runs. +################################################################################ + +seed 12345 +dimension 3 +global gridcut 1.0e-5 comm/sort yes + +boundary rr rr rr + +create_box 0 0.0001 0 0.0001 0 0.0001 +Created orthogonal box = (0 0 0) to (0.0001 0.0001 0.0001) +create_grid 4 4 4 +Created 64 child grid cells + CPU time = 0.000850154 secs + create/ghost percent = 95.0951 4.90488 +balance_grid rcb part +Balance grid migrated 0 cells + CPU time = 0.000138633 secs + reassign/sort/migrate/ghost percent = 83.413 0.0880021 10.038 6.46094 + +species ar.species Ar +mixture air Ar vstream 0.0 0.0 0.0 temp 273.15 + +global nrho 7.07043E22 fnum 7.07043E7 + +create_particles air n 2000 twopass +Created 2000 particles + CPU time = 0.00109105 secs + +compute temp temp +stats 10 +stats_style step np c_temp + +timestep 7.00E-9 + +variable np equal extract_setting(world_size) +restart 10 tmp.free.periodic.np${np}.*.equil.restart +restart 10 tmp.free.periodic.np1.*.equil.restart +run 20 +Memory usage per proc in Mbytes: + particles (ave,min,max) = 1.5625 1.5625 1.5625 + grid (ave,min,max) = 1.51379 1.51379 1.51379 + surf (ave,min,max) = 0 0 0 + modify (ave,min,max) = 0 0 0 + total (ave,min,max) = 3.07629 3.07629 3.07629 +Step Np c_temp + 0 2000 274.64074 + 10 2000 274.64074 + 20 2000 274.64074 +Loop time of 0.000867992 on 1 procs for 20 steps with 2000 particles +Performance: 23041.687 timesteps/s, 46.083 Mparticle-step/s + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Move | 0.00040587 | 0.00040587 | 0.00040587 | 0.0 | 46.76 +Coll | 0 | 0 | 0 | 0.0 | 0.00 +Sort | 0 | 0 | 0 | 0.0 | 0.00 +Comm | 7.649e-06 | 7.649e-06 | 7.649e-06 | 0.0 | 0.88 +Modify | 0 | 0 | 0 | 0.0 | 0.00 +Output | 0.00045024 | 0.00045024 | 0.00045024 | 0.0 | 51.87 +MPI Sync| 2.213e-06 | 2.213e-06 | 2.213e-06 | 0.0 | 0.25 +Other | | 2.016e-06 | | | 0.23 + +Particle moves = 40000 (40K) +Cells touched = 44761 (44.8K) +Particle comms = 0 (0K) +Boundary collides = 1605 (1.6K) +Boundary exits = 0 (0K) +SurfColl checks = 0 (0K) +SurfColl occurs = 0 (0K) +Surf reactions = 0 (0K) +Collide attempts = 0 (0K) +Collide occurs = 0 (0K) +Reactions = 0 (0K) +Particles stuck = 0 +Axisymm bad moves = 0 + +Particle-moves/CPUsec/proc: 4.60834e+07 +Particle-moves/step: 2000 +Cell-touches/particle/step: 1.11902 +Particle comm iterations/step: 1 +Particle fraction communicated: 0 +Particle fraction colliding with boundary: 0.040125 +Particle fraction exiting boundary: 0 +Surface-checks/particle/step: 0 +Surface-collisions/particle/step: 0 +Surf-reactions/particle/step: 0 +Collision-attempts/particle/step: 0 +Collisions/particle/step: 0 +Reactions/particle/step: 0 + +Particles: 2000 ave 2000 max 2000 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Cells: 64 ave 64 max 64 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +GhostCell: 0 ave 0 max 0 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +EmptyCell: 0 ave 0 max 0 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +write_restart tmp.free.np${np}.*.equil.restart +write_restart tmp.free.np1.*.equil.restart + +# read the restart back through the "*" wildcard search and continue + +clear +Running on 1 MPI task(s) +seed 12345 +variable np equal extract_setting(world_size) +read_restart tmp.free.np${np}.*.equil.restart +read_restart tmp.free.np1.*.equil.restart + orthogonal box = (0 0 0) to (0.0001 0.0001 0.0001) + 64 grid cells + 2000 particles + CPU time = 0.00171303 secs + read/surf2grid/rebalance/ghost/inout percent = 98.922 0.01366 0.00402796 1.05713 0.00315231 + +compute temp temp +stats 10 +stats_style step np c_temp +run 20 +Memory usage per proc in Mbytes: + particles (ave,min,max) = 1.5625 1.5625 1.5625 + grid (ave,min,max) = 1.51379 1.51379 1.51379 + surf (ave,min,max) = 0 0 0 + modify (ave,min,max) = 0 0 0 + total (ave,min,max) = 3.07629 3.07629 3.07629 +Step Np c_temp + 20 2000 274.64074 + 30 2000 274.64074 + 40 2000 274.64074 +Loop time of 0.000458225 on 1 procs for 20 steps with 2000 particles +Performance: 43646.680 timesteps/s, 87.293 Mparticle-step/s + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Move | 0.00041485 | 0.00041485 | 0.00041485 | 0.0 | 90.53 +Coll | 0 | 0 | 0 | 0.0 | 0.00 +Sort | 0 | 0 | 0 | 0.0 | 0.00 +Comm | 6.637e-06 | 6.637e-06 | 6.637e-06 | 0.0 | 1.45 +Modify | 0 | 0 | 0 | 0.0 | 0.00 +Output | 3.3252e-05 | 3.3252e-05 | 3.3252e-05 | 0.0 | 7.26 +MPI Sync| 2.173e-06 | 2.173e-06 | 2.173e-06 | 0.0 | 0.47 +Other | | 1.318e-06 | | | 0.29 + +Particle moves = 40000 (40K) +Cells touched = 44780 (44.8K) +Particle comms = 0 (0K) +Boundary collides = 1647 (1.65K) +Boundary exits = 0 (0K) +SurfColl checks = 0 (0K) +SurfColl occurs = 0 (0K) +Surf reactions = 0 (0K) +Collide attempts = 0 (0K) +Collide occurs = 0 (0K) +Reactions = 0 (0K) +Particles stuck = 0 +Axisymm bad moves = 0 + +Particle-moves/CPUsec/proc: 8.72934e+07 +Particle-moves/step: 2000 +Cell-touches/particle/step: 1.1195 +Particle comm iterations/step: 1 +Particle fraction communicated: 0 +Particle fraction colliding with boundary: 0.041175 +Particle fraction exiting boundary: 0 +Surface-checks/particle/step: 0 +Surface-collisions/particle/step: 0 +Surf-reactions/particle/step: 0 +Collision-attempts/particle/step: 0 +Collisions/particle/step: 0 +Reactions/particle/step: 0 + +Particles: 2000 ave 2000 max 2000 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Cells: 64 ave 64 max 64 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +GhostCell: 0 ave 0 max 0 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +EmptyCell: 0 ave 0 max 0 min +Histogram: 1 0 0 0 0 0 0 0 0 0 diff --git a/examples/free/log.22Aug26.mpi_4.free b/examples/free/log.22Aug26.mpi_4.free new file mode 100644 index 000000000..1b66ec5a9 --- /dev/null +++ b/examples/free/log.22Aug26.mpi_4.free @@ -0,0 +1,130 @@ +SPARTA (24 Sep 2025) +Running on 4 MPI task(s) +################################################################################ +# thermal gas in a 3d box with free molecular flow (no collisions) +# +# Note: +# - The "comm/sort” option to the “global” command is used to match MPI runs. +# - The “twopass” option is used to match Kokkos runs. +# The "comm/sort" and "twopass" options should not be used for production runs. +################################################################################ +# particles reflect off global box boundaries + +seed 12345 +dimension 3 +global gridcut 1.0e-5 comm/sort yes + +boundary rr rr rr + +create_box 0 0.0001 0 0.0001 0 0.0001 +Created orthogonal box = (0 0 0) to (0.0001 0.0001 0.0001) +create_grid 10 10 10 +WARNING: Could not acquire nearby ghost cells b/c grid partition is not clumped (/home/user/sparta/src/grid.cpp:486) +Created 1000 child grid cells + CPU time = 0.00102479 secs + create/ghost percent = 92.0182 7.9818 + +balance_grid rcb part +Balance grid migrated 740 cells + CPU time = 0.000639433 secs + reassign/sort/migrate/ghost percent = 45.8145 0.375333 31.0391 22.7711 + +species ar.species Ar +mixture air Ar vstream 0.0 0.0 0.0 temp 273.15 + +global nrho 7.07043E22 +global fnum 7.07043E6 + +create_particles air n 10000 twopass +Created 10000 particles + CPU time = 0.00136636 secs + +stats 100 +compute temp temp + +# compute boundary drives the boundary-tally path in Update; every box face +# here is reflecting, so all six tally. compute ke/particle is reduced to a +# scalar so it can appear in stats. + +# every box face here is specular (rr), which conserves energy and applies no +# shear, so ke/erot/evib and shx/shy/shz would be identically zero and could +# not detect a regression. n/nwt/press are the values that actually vary. + +compute bound boundary all n nwt press + +stats_style step cpu np nattempt ncoll c_temp c_bound[1][1] c_bound[1][3] c_bound[4][3] c_bound[6][3] + +#dump 2 image all 100 image.*.ppm type type pdiam 3.0e-6 # size 512 512 gline yes 0.005 +#dump_modify 2 pad 4 + +timestep 7.00E-9 +run 1000 +Memory usage per proc in Mbytes: + particles (ave,min,max) = 1.5625 1.5625 1.5625 + grid (ave,min,max) = 1.51379 1.51379 1.51379 + surf (ave,min,max) = 0 0 0 + modify (ave,min,max) = 0 0 0 + total (ave,min,max) = 3.07629 3.07629 3.07629 +Step CPU Np Natt Ncoll c_temp c_bound[1][1] c_bound[1][3] c_bound[4][3] c_bound[6][3] + 0 0 10000 0 0 274.13466 0 0 0 0 + 100 0.006658057 10000 0 0 274.13466 67 280.47257 310.57573 303.11802 + 200 0.014053502 10000 0 0 274.13466 69 266.48541 189.10393 339.2908 + 300 0.020794501 10000 0 0 274.13466 56 253.65262 217.85743 237.829 + 400 0.029173644 10000 0 0 274.13466 58 214.76266 295.1197 297.05966 + 500 0.035688348 10000 0 0 274.13466 67 288.43361 248.34494 277.43574 + 600 0.041702235 10000 0 0 274.13466 72 280.85757 230.33553 249.15976 + 700 0.047866879 10000 0 0 274.13466 77 292.12969 279.57766 228.53605 + 800 0.054243475 10000 0 0 274.13466 64 245.09374 241.32478 284.12465 + 900 0.062382927 10000 0 0 274.13466 56 223.97301 257.58684 335.97284 + 1000 0.069240341 10000 0 0 274.13466 77 294.09322 231.30629 318.03375 +Loop time of 0.069314 on 4 procs for 1000 steps with 10000 particles +Performance: 14427.093 timesteps/s, 144.271 Mparticle-step/s + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Move | 0.042358 | 0.04421 | 0.045639 | 0.6 | 63.78 +Coll | 0 | 0 | 0 | 0.0 | 0.00 +Sort | 0 | 0 | 0 | 0.0 | 0.00 +Comm | 0.012855 | 0.013225 | 0.013785 | 0.3 | 19.08 +Modify | 0 | 0 | 0 | 0.0 | 0.00 +Output | 0.00031984 | 0.00045113 | 0.00084144 | 0.0 | 0.65 +MPI Sync| 0.010032 | 0.011338 | 0.013173 | 1.1 | 16.36 +Other | | 9.023e-05 | | | 0.13 + +Particle moves = 10000000 (10M) +Cells touched = 13601534 (13.6M) +Particle comms = 264033 (0.264M) +Boundary collides = 400226 (0.4M) +Boundary exits = 0 (0K) +SurfColl checks = 0 (0K) +SurfColl occurs = 0 (0K) +Surf reactions = 0 (0K) +Collide attempts = 0 (0K) +Collide occurs = 0 (0K) +Reactions = 0 (0K) +Particles stuck = 0 +Axisymm bad moves = 0 + +Particle-moves/CPUsec/proc: 3.60677e+07 +Particle-moves/step: 10000 +Cell-touches/particle/step: 1.36015 +Particle comm iterations/step: 1 +Particle fraction communicated: 0.0264033 +Particle fraction colliding with boundary: 0.0400226 +Particle fraction exiting boundary: 0 +Surface-checks/particle/step: 0 +Surface-collisions/particle/step: 0 +Surf-reactions/particle/step: 0 +Collision-attempts/particle/step: 0 +Collisions/particle/step: 0 +Reactions/particle/step: 0 + +Particles: 2500 ave 2571 max 2423 min +Histogram: 1 0 0 0 0 2 0 0 0 1 +Cells: 250 ave 250 max 250 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +GhostCell: 110 ave 110 max 110 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +EmptyCell: 0 ave 0 max 0 min +Histogram: 4 0 0 0 0 0 0 0 0 0 diff --git a/examples/free/log.22Aug26.mpi_4.free.restart b/examples/free/log.22Aug26.mpi_4.free.restart new file mode 100644 index 000000000..227a5242e --- /dev/null +++ b/examples/free/log.22Aug26.mpi_4.free.restart @@ -0,0 +1,208 @@ +SPARTA (24 Sep 2025) +Running on 4 MPI task(s) +################################################################################ +# restart round trip: write a restart file, read it back, continue the run +# +# This is the only deck that exercises read_restart in CI -- the four +# examples/custom/in.custom.*.restart decks are all in SPARTA_DISABLED_TESTS. +# +# All three filename-substitution paths are driven deliberately: +# restart -> Output::write() (periodic restart output) +# write_restart -> WriteRestart::command() +# read_restart -> ReadRestart::file_search() (the "*" wildcard search) +# Each replaces "*" with a timestep, and each sizes the substituted name +# itself. The ".equil.restart" suffix is deliberately long: a name is +# truncated only when (timestep digits + suffix length) exceeds 15, so a +# short suffix would let a sizing bug through unnoticed. +# +# The file name carries the MPI rank count, because ctest runs the mpi_1 and +# mpi_4 copies of a test concurrently in one shared suite directory and they +# would otherwise write the same file (compare the note on +# in.exp2imp.axi.spherecone.readback in SPARTA_DISABLED_TESTS). +# +# Note: +# - The "comm/sort" option to the "global" command is used to match MPI runs. +# - The "twopass" option is used to match Kokkos runs. +# The "comm/sort" and "twopass" options should not be used for production runs. +################################################################################ + +seed 12345 +dimension 3 +global gridcut 1.0e-5 comm/sort yes + +boundary rr rr rr + +create_box 0 0.0001 0 0.0001 0 0.0001 +Created orthogonal box = (0 0 0) to (0.0001 0.0001 0.0001) +create_grid 4 4 4 +WARNING: Could not acquire nearby ghost cells b/c grid partition is not clumped (/home/user/sparta/src/grid.cpp:486) +Created 64 child grid cells + CPU time = 0.00127803 secs + create/ghost percent = 90.4759 9.52413 +balance_grid rcb part +Balance grid migrated 32 cells + CPU time = 0.000376637 secs + reassign/sort/migrate/ghost percent = 67.183 0.503668 8.99407 23.3193 + +species ar.species Ar +mixture air Ar vstream 0.0 0.0 0.0 temp 273.15 + +global nrho 7.07043E22 fnum 7.07043E7 + +create_particles air n 2000 twopass +Created 2000 particles + CPU time = 0.00125952 secs + +compute temp temp +stats 10 +stats_style step np c_temp + +timestep 7.00E-9 + +variable np equal extract_setting(world_size) +restart 10 tmp.free.periodic.np${np}.*.equil.restart +restart 10 tmp.free.periodic.np4.*.equil.restart +run 20 +Memory usage per proc in Mbytes: + particles (ave,min,max) = 1.5625 1.5625 1.5625 + grid (ave,min,max) = 1.51379 1.51379 1.51379 + surf (ave,min,max) = 0 0 0 + modify (ave,min,max) = 0 0 0 + total (ave,min,max) = 3.07629 3.07629 3.07629 +Step Np c_temp + 0 2000 261.46165 + 10 2000 261.46165 + 20 2000 261.46165 +Loop time of 0.000777178 on 4 procs for 20 steps with 2000 particles +Performance: 25734.130 timesteps/s, 51.468 Mparticle-step/s + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Move | 0.00010688 | 0.00011367 | 0.00011945 | 0.0 | 14.63 +Coll | 0 | 0 | 0 | 0.0 | 0.00 +Sort | 0 | 0 | 0 | 0.0 | 0.00 +Comm | 0.00020576 | 0.00020971 | 0.00021433 | 0.0 | 26.98 +Modify | 0 | 0 | 0 | 0.0 | 0.00 +Output | 0.00035802 | 0.00036892 | 0.00040113 | 0.0 | 47.47 +MPI Sync| 5.354e-05 | 8.3034e-05 | 9.3857e-05 | 0.0 | 10.68 +Other | | 1.851e-06 | | | 0.24 + +Particle moves = 40000 (40K) +Cells touched = 44638 (44.6K) +Particle comms = 1042 (1.04K) +Boundary collides = 1534 (1.53K) +Boundary exits = 0 (0K) +SurfColl checks = 0 (0K) +SurfColl occurs = 0 (0K) +Surf reactions = 0 (0K) +Collide attempts = 0 (0K) +Collide occurs = 0 (0K) +Reactions = 0 (0K) +Particles stuck = 0 +Axisymm bad moves = 0 + +Particle-moves/CPUsec/proc: 1.28671e+07 +Particle-moves/step: 2000 +Cell-touches/particle/step: 1.11595 +Particle comm iterations/step: 1 +Particle fraction communicated: 0.02605 +Particle fraction colliding with boundary: 0.03835 +Particle fraction exiting boundary: 0 +Surface-checks/particle/step: 0 +Surface-collisions/particle/step: 0 +Surf-reactions/particle/step: 0 +Collision-attempts/particle/step: 0 +Collisions/particle/step: 0 +Reactions/particle/step: 0 + +Particles: 500 ave 525 max 487 min +Histogram: 1 2 0 0 0 0 0 0 0 1 +Cells: 16 ave 16 max 16 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +GhostCell: 20 ave 20 max 20 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +EmptyCell: 0 ave 0 max 0 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +write_restart tmp.free.np${np}.*.equil.restart +write_restart tmp.free.np4.*.equil.restart + +# read the restart back through the "*" wildcard search and continue + +clear +Running on 4 MPI task(s) +seed 12345 +variable np equal extract_setting(world_size) +read_restart tmp.free.np${np}.*.equil.restart +read_restart tmp.free.np4.*.equil.restart + orthogonal box = (0 0 0) to (0.0001 0.0001 0.0001) + 64 grid cells + 2000 particles + CPU time = 0.00185424 secs + read/surf2grid/rebalance/ghost/inout percent = 97.1426 0.13035 0.0470272 2.62867 0.0513956 + +compute temp temp +stats 10 +stats_style step np c_temp +run 20 +Memory usage per proc in Mbytes: + particles (ave,min,max) = 1.5625 1.5625 1.5625 + grid (ave,min,max) = 1.51379 1.51379 1.51379 + surf (ave,min,max) = 0 0 0 + modify (ave,min,max) = 0 0 0 + total (ave,min,max) = 3.07629 3.07629 3.07629 +Step Np c_temp + 20 2000 261.46165 + 30 2000 261.46165 + 40 2000 261.46165 +Loop time of 0.000284931 on 4 procs for 20 steps with 2000 particles +Performance: 70192.371 timesteps/s, 140.385 Mparticle-step/s + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Move | 9.8676e-05 | 0.00010795 | 0.00011825 | 0.0 | 37.89 +Coll | 0 | 0 | 0 | 0.0 | 0.00 +Sort | 0 | 0 | 0 | 0.0 | 0.00 +Comm | 0.00012095 | 0.00012507 | 0.00013182 | 0.0 | 43.90 +Modify | 0 | 0 | 0 | 0.0 | 0.00 +Output | 8.549e-06 | 1.0682e-05 | 1.494e-05 | 0.0 | 3.75 +MPI Sync| 3.2964e-05 | 4e-05 | 4.7641e-05 | 0.0 | 14.04 +Other | | 1.228e-06 | | | 0.43 + +Particle moves = 40000 (40K) +Cells touched = 44739 (44.7K) +Particle comms = 1035 (1.03K) +Boundary collides = 1606 (1.61K) +Boundary exits = 0 (0K) +SurfColl checks = 0 (0K) +SurfColl occurs = 0 (0K) +Surf reactions = 0 (0K) +Collide attempts = 0 (0K) +Collide occurs = 0 (0K) +Reactions = 0 (0K) +Particles stuck = 0 +Axisymm bad moves = 0 + +Particle-moves/CPUsec/proc: 3.50962e+07 +Particle-moves/step: 2000 +Cell-touches/particle/step: 1.11848 +Particle comm iterations/step: 1 +Particle fraction communicated: 0.025875 +Particle fraction colliding with boundary: 0.04015 +Particle fraction exiting boundary: 0 +Surface-checks/particle/step: 0 +Surface-collisions/particle/step: 0 +Surf-reactions/particle/step: 0 +Collision-attempts/particle/step: 0 +Collisions/particle/step: 0 +Reactions/particle/step: 0 + +Particles: 500 ave 520 max 491 min +Histogram: 1 2 0 0 0 0 0 0 0 1 +Cells: 16 ave 16 max 16 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +GhostCell: 20 ave 20 max 20 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +EmptyCell: 0 ave 0 max 0 min +Histogram: 4 0 0 0 0 0 0 0 0 0 diff --git a/examples/qk/air.species b/examples/qk/air.species new file mode 100644 index 000000000..d102d9070 --- /dev/null +++ b/examples/qk/air.species @@ -0,0 +1,24 @@ +# Species data + +# ID +# Molwt (amu) +# Molmass (kg) +# Rotational dof +# RotRel +# Vibrational dof +# VibRel +# VibTemp (K) +# species wt +# charge + +O2 32.00 5.31E-26 2 0.2 2 5.58659E-5 2256.0 1.0 0.0 +N2 28.016 4.65E-26 2 0.2 2 1.90114E-5 3371.0 1.0 0.0 +O 16.00 2.65E-26 0 0.0 0 0.0 0.0 1.0 0.0 +N 14.008 2.325E-26 0 0.0 0 0.0 0.0 1.0 0.0 +NO 30.008 4.98E-26 2 0.2 2 7.14285E-4 2719.0 1.0 0.0 +O2+ 32.00 5.31E-26 2 0.2 2 5.58659E-5 2256.0 1.0 1.0 +N2+ 28.016 4.65E-26 2 0.2 2 1.90114E-5 3371.0 1.0 1.0 +O+ 16.00 2.65E-26 0 0.0 0 0.0 0.0 1.0 1.0 +N+ 14.008 2.325E-26 0 0.0 0 0.0 0.0 1.0 1.0 +NO+ 30.008 4.98E-26 2 0.2 2 7.14285E-4 2719.0 1.0 1.0 +e 0.001 9.10938188E-31 0 0.0 0 0.0 0.0 1.0 -1.0 diff --git a/examples/qk/air.tce b/examples/qk/air.tce new file mode 100644 index 000000000..93961c3a5 --- /dev/null +++ b/examples/qk/air.tce @@ -0,0 +1,136 @@ +# reactions in air + +O2 + N --> O + O + N +D A 1.0 8.197e-19 1.660e-8 -1.5 -8.197e-19 + +O2 + NO --> O + O + NO +D A 1.0 8.197e-19 3.321e-9 -1.5 -8.197e-19 + +O2 + N2 --> O + O + N2 +D A 1.0 8.197e-19 3.321e-9 -1.5 -8.197e-19 + +O2 + O2 --> O + O + O2 +D A 1.0 8.197e-19 3.321e-9 -1.5 -8.197e-19 + +O2 + O --> O + O + O +D A 1.0 8.197e-19 1.660e-8 -1.5 -8.197e-19 + +N2 + O --> N + N + O +D A 1.0 1.561e-18 4.980e-8 -1.6 -1.561e-18 + +N2 + O2 --> N + N + O2 +D A 1.0 1.561e-18 1.162e-8 -1.6 -1.561e-18 + +N2 + NO --> N + N + NO +D A 1.0 1.561e-18 1.162e-8 -1.6 -1.561e-18 + +N2 + N2 --> N + N + N2 +D A 1.0 1.561e-18 1.162e-8 -1.6 -1.561e-18 + +N2 + N --> N + N + N +D A 1.0 1.561e-18 4.980e-8 -1.6 -1.561e-18 + +NO + N2 --> N + O + N2 +D A 1.0 1.043e-18 8.302e-15 0.00 -1.043e-18 + +NO + O2 --> N + O + O2 +D A 1.0 1.043e-18 8.302e-15 0.00 -1.043e-18 + +NO + NO --> N + O + NO +D A 1.0 1.043e-18 8.302e-15 0.00 -1.043e-18 + +NO + O --> N + O + O +D A 1.0 1.043e-18 1.862e-13 0.0 -1.043e-18 + +NO + N --> N + O + N +D A 1.0 1.043e-18 1.862e-13 0.0 -1.043e-18 + +NO + O --> O2 + N +E A 0.0 2.684e-19 1.389e-17 0.0 -2.684e-19 + +N2 + O --> NO + N +E A 0.0 5.175e-19 1.069e-12 -1.0 -5.175e-19 + +O2 + N --> NO + O +E A 0.0 0.0 4.601e-15 -0.546 2.684e-19 + +NO + N --> N2 + O +E A 0.0 0.0 4.059e-12 -1.359 5.175e-19 + +O + N --> NO+ + e +I A 0.0 4.404e-19 8.766e-18 0.0 -4.404e-19 + +N + N --> N2+ + e +I A 0.0 9.319e-19 3.387e-17 0.0 -9.319e-19 + +O + O --> O2+ + e +I A 0.0 1.1128e-18 1.8580e-17 0.0 -1.1128e-18 + +NO+ + N --> O + N2+ +E A 0.0 4.832e-19 1.1956e-16 0.0 -4.832e-19 + +N2+ + O --> N + NO+ +E A 0.0 0.0000 1.744e-18 0.302 4.832e-19 + +N2 + N+ --> N + N2+ +E A 0.0 1.684e-19 1.6605e-18 0.5 -1.684e-19 + +N2+ + N --> N2 + N+ +E A 0.0 0.0000000 1.295e-18 0.5 1.684e-19 + +NO+ + N --> N2 + O+ +E A 0.0 1.767e-19 5.6458e-17 1.08 -1.767e-19 + +N2 + O+ --> N + NO+ +E A 0.0 0.0000000 3.9708e-18 -0.710 1.767e-19 + +NO+ + O --> O2 + N+ +E A 0.0 1.767e-19 1.6605e-18 0.5 -1.767e-19 + +O2 + N+ --> O + NO+ +E A 0.0 0.0000000 3.040e-18 -0.29 1.767e-19 + +NO+ + O --> N + O2+ +E A 0.0 6.710e-19 1.1956e-17 0.29 -6.710e-19 + +O2+ + N --> O + NO+ +E A 0.0 0.0000000 8.918e-13 -0.969 6.710e-19 + +NO+ + O2 --> NO + O2+ +E A 0.0 4.501e-19 3.9853e-17 0.41 -4.501e-19 + +O2+ + NO --> O2 + NO+ +E A 0.0 0.0000000 3.990e-17 0.41 4.501e-19 + +O2+ + N --> O2 + N+ +E A 0.0 3.949e-19 1.4447e-16 0.14 -3.949e-19 + +O2+ + O --> O2 + O+ +E A 0.0 2.485e-19 6.6422e-18 -0.09 -2.485e-19 + +O+ + O2 --> O + O2+ +E A 0.0 0.0000000 4.993e-18 -0.004 2.485e-19 + +O2+ + N2 --> O2 + N2+ +E A 0.0 5.619e-19 1.6439e-17 0.00 -5.619e-19 + +N2+ + O2 --> N2 + O2+ +E A 0.0 0.0000000 4.5899e-18 -0.037 5.619e-19 + +O+ + N2 --> O + N2+ +E A 0.0 3.148e-19 1.5111e-18 0.00 -1.148e-19 + +N2+ + O --> N2 + O+ +E A 0.0 0.000 4.118e-11 -2.2 1.148e-19 + +O+ + NO --> O2 + N+ +E A 0.0 3.673e-19 2.3248e-25 1.90 -3.673e-19 + +N+ + O2 --> NO + O+ +E A 0.0 0.000 2.443e-26 2.102 3.673e-19 + +O + e --> O+ + e + e +I A 0.0 2.188e-18 6.4761E3 -3.78 -2.188e-18 + +N + e --> N+ + e + e +I A 0.0 2.322e-18 4.1513E4 -3.82 -2.322e-18 diff --git a/examples/qk/air.vss b/examples/qk/air.vss new file mode 100644 index 000000000..42ddebb0d --- /dev/null +++ b/examples/qk/air.vss @@ -0,0 +1,18 @@ +# VSS collision model parameters for each species + +# diameter (m) +# omega +# tref +# alpha + +O2 3.96E-10 0.77 273.15 1.4 +N2 4.07E-10 0.74 273.15 1.6 +O 3.0E-10 0.80 273.15 1.0 +N 3.0E-10 0.80 273.15 1.0 +NO 4.0E-10 0.80 273.15 1.0 +O2+ 3.96E-10 0.77 273.15 1.4 +N2+ 4.07E-10 0.74 273.15 1.6 +O+ 3.0E-10 0.80 273.15 1.0 +N+ 3.0E-10 0.80 273.15 1.0 +NO+ 4.0E-10 0.80 273.15 1.0 +e 7.0E-13 0.50 273.15 1.0 diff --git a/examples/qk/in.qk b/examples/qk/in.qk new file mode 100644 index 000000000..e6deb8f87 --- /dev/null +++ b/examples/qk/in.qk @@ -0,0 +1,44 @@ +################################################################################ +# thermal gas in a 3d box with collisions and chemistry +# particles reflect off global box boundaries +# +# Note: +# - The "comm/sort” option to the “global” command is used to match MPI runs. +# - The “twopass” option is used to match Kokkos runs. +# The "comm/sort" and "twopass" options should not be used for production runs. +################################################################################ + +seed 12345 +dimension 3 +global gridcut 1.0e-5 comm/sort yes + +boundary rr rr rr + +create_box 0 0.0001 0 0.0001 0 0.0001 +create_grid 10 10 10 + +balance_grid rcb part + +species air.species N2 N +mixture air N2 N vstream 0.0 0.0 0.0 temp 20000.0 +mixture air N2 frac 1.0 +mixture air N frac 0.0 + +global nrho 7.07043E22 +global fnum 7.07043E5 + +collide vss air air.vss +react qk air.tce + +create_particles air n 10000 twopass + +stats 100 +compute temp temp +stats_style step cpu np nattempt ncoll nreact c_temp + +#dump 2 image all 100 image.*.ppm type type pdiam 3.0e-6 & +# size 512 512 gline yes 0.005 +#dump_modify 2 pad 4 + +timestep 7.00E-9 +run 400 diff --git a/examples/qk/log.22Aug26.mpi_1.qk b/examples/qk/log.22Aug26.mpi_1.qk new file mode 100644 index 000000000..ea125aa98 --- /dev/null +++ b/examples/qk/log.22Aug26.mpi_1.qk @@ -0,0 +1,122 @@ +SPARTA (24 Sep 2025) +Running on 1 MPI task(s) +################################################################################ +# thermal gas in a 3d box with collisions and chemistry +# particles reflect off global box boundaries +# +# Note: +# - The "comm/sort” option to the “global” command is used to match MPI runs. +# - The “twopass” option is used to match Kokkos runs. +# The "comm/sort" and "twopass" options should not be used for production runs. +################################################################################ + +seed 12345 +dimension 3 +global gridcut 1.0e-5 comm/sort yes + +boundary rr rr rr + +create_box 0 0.0001 0 0.0001 0 0.0001 +Created orthogonal box = (0 0 0) to (0.0001 0.0001 0.0001) +create_grid 10 10 10 +Created 1000 child grid cells + CPU time = 0.000914428 secs + create/ghost percent = 79.8424 20.1576 + +balance_grid rcb part +Balance grid migrated 0 cells + CPU time = 0.000233165 secs + reassign/sort/migrate/ghost percent = 53.4488 0.436601 8.74059 37.374 + +species air.species N2 N +mixture air N2 N vstream 0.0 0.0 0.0 temp 20000.0 +mixture air N2 frac 1.0 +mixture air N frac 0.0 + +global nrho 7.07043E22 +global fnum 7.07043E5 + +collide vss air air.vss +react qk air.tce + +create_particles air n 10000 twopass +Created 10000 particles + CPU time = 0.00199929 secs + +stats 100 +compute temp temp +stats_style step cpu np nattempt ncoll nreact c_temp + +#dump 2 image all 100 image.*.ppm type type pdiam 3.0e-6 # size 512 512 gline yes 0.005 +#dump_modify 2 pad 4 + +timestep 7.00E-9 +run 400 +Memory usage per proc in Mbytes: + particles (ave,min,max) = 1.5625 1.5625 1.5625 + grid (ave,min,max) = 1.51379 1.51379 1.51379 + surf (ave,min,max) = 0 0 0 + modify (ave,min,max) = 0 0 0 + total (ave,min,max) = 3.07629 3.07629 3.07629 +Step CPU Np Natt Ncoll Nreact c_temp + 0 0 10000 0 0 0 19907.187 + 100 0.14348323 10258 987 255 1 18364.582 + 200 0.29238392 10445 1038 250 0 17229.34 + 300 0.44129555 10585 1076 277 4 16588.006 + 400 0.5978651 10691 1078 259 1 16253.022 +Loop time of 0.597924 on 1 procs for 400 steps with 10691 particles +Performance: 668.982 timesteps/s, 7.152 Mparticle-step/s + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Move | 0.48535 | 0.48535 | 0.48535 | 0.0 | 81.17 +Coll | 0.099718 | 0.099718 | 0.099718 | 0.0 | 16.68 +Sort | 0.011562 | 0.011562 | 0.011562 | 0.0 | 1.93 +Comm | 0.00043256 | 0.00043256 | 0.00043256 | 0.0 | 0.07 +Modify | 0 | 0 | 0 | 0.0 | 0.00 +Output | 0.00036631 | 0.00036631 | 0.00036631 | 0.0 | 0.06 +MPI Sync| 0.00043601 | 0.00043601 | 0.00043601 | 0.0 | 0.07 +Other | | 6.095e-05 | | | 0.01 + +Particle moves = 4164928 (4.16M) +Cells touched = 18972473 (19M) +Particle comms = 0 (0K) +Boundary collides = 1645138 (1.65M) +Boundary exits = 0 (0K) +SurfColl checks = 0 (0K) +SurfColl occurs = 0 (0K) +Surf reactions = 0 (0K) +Collide attempts = 410177 (0.41M) +Collide occurs = 102844 (0.103M) +Reactions = 691 (0.691K) +Particles stuck = 0 +Axisymm bad moves = 0 + +Particle-moves/CPUsec/proc: 6.96565e+06 +Particle-moves/step: 10412.3 +Cell-touches/particle/step: 4.55529 +Particle comm iterations/step: 1 +Particle fraction communicated: 0 +Particle fraction colliding with boundary: 0.394998 +Particle fraction exiting boundary: 0 +Surface-checks/particle/step: 0 +Surface-collisions/particle/step: 0 +Surf-reactions/particle/step: 0 +Collision-attempts/particle/step: 0.0984836 +Collisions/particle/step: 0.0246929 +Reactions/particle/step: 0.000165909 + +Gas reaction tallies: + style qk #-of-reactions 45 + reaction N2 + N2 --> N + N + N2: 665 + reaction N2 + N --> N + N + N: 26 + +Particles: 10691 ave 10691 max 10691 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Cells: 1000 ave 1000 max 1000 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +GhostCell: 0 ave 0 max 0 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +EmptyCell: 0 ave 0 max 0 min +Histogram: 1 0 0 0 0 0 0 0 0 0 diff --git a/examples/qk/log.22Aug26.mpi_4.qk b/examples/qk/log.22Aug26.mpi_4.qk new file mode 100644 index 000000000..d141736ed --- /dev/null +++ b/examples/qk/log.22Aug26.mpi_4.qk @@ -0,0 +1,123 @@ +SPARTA (24 Sep 2025) +Running on 4 MPI task(s) +################################################################################ +# thermal gas in a 3d box with collisions and chemistry +# particles reflect off global box boundaries +# +# Note: +# - The "comm/sort” option to the “global” command is used to match MPI runs. +# - The “twopass” option is used to match Kokkos runs. +# The "comm/sort" and "twopass" options should not be used for production runs. +################################################################################ + +seed 12345 +dimension 3 +global gridcut 1.0e-5 comm/sort yes + +boundary rr rr rr + +create_box 0 0.0001 0 0.0001 0 0.0001 +Created orthogonal box = (0 0 0) to (0.0001 0.0001 0.0001) +create_grid 10 10 10 +WARNING: Could not acquire nearby ghost cells b/c grid partition is not clumped (/home/user/sparta/src/grid.cpp:486) +Created 1000 child grid cells + CPU time = 0.0318488 secs + create/ghost percent = 14.5166 85.4834 + +balance_grid rcb part +Balance grid migrated 740 cells + CPU time = 0.000810853 secs + reassign/sort/migrate/ghost percent = 45.1264 0.315224 15.0967 39.4617 + +species air.species N2 N +mixture air N2 N vstream 0.0 0.0 0.0 temp 20000.0 +mixture air N2 frac 1.0 +mixture air N frac 0.0 + +global nrho 7.07043E22 +global fnum 7.07043E5 + +collide vss air air.vss +react qk air.tce + +create_particles air n 10000 twopass +Created 10000 particles + CPU time = 0.00291076 secs + +stats 100 +compute temp temp +stats_style step cpu np nattempt ncoll nreact c_temp + +#dump 2 image all 100 image.*.ppm type type pdiam 3.0e-6 # size 512 512 gline yes 0.005 +#dump_modify 2 pad 4 + +timestep 7.00E-9 +run 400 +Memory usage per proc in Mbytes: + particles (ave,min,max) = 1.5625 1.5625 1.5625 + grid (ave,min,max) = 1.51379 1.51379 1.51379 + surf (ave,min,max) = 0 0 0 + modify (ave,min,max) = 0 0 0 + total (ave,min,max) = 3.07629 3.07629 3.07629 +Step CPU Np Natt Ncoll Nreact c_temp + 0 0 10000 0 0 0 19847.392 + 100 0.045653648 10280 997 249 2 18282.13 + 200 0.088671906 10450 1027 253 1 17342.412 + 300 0.13204167 10592 1046 259 1 16858.714 + 400 0.17290921 10700 1068 288 1 16200.268 +Loop time of 0.172987 on 4 procs for 400 steps with 10700 particles +Performance: 2312.310 timesteps/s, 24.742 Mparticle-step/s + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Move | 0.11996 | 0.12247 | 0.12586 | 0.6 | 70.80 +Coll | 0.01427 | 0.01485 | 0.015417 | 0.3 | 8.58 +Sort | 0.0014981 | 0.0015783 | 0.001685 | 0.2 | 0.91 +Comm | 0.020717 | 0.020932 | 0.02108 | 0.1 | 12.10 +Modify | 0 | 0 | 0 | 0.0 | 0.00 +Output | 9.0234e-05 | 0.00016105 | 0.00036074 | 0.0 | 0.09 +MPI Sync| 0.0091846 | 0.012973 | 0.01625 | 2.2 | 7.50 +Other | | 2.405e-05 | | | 0.01 + +Particle moves = 4168874 (4.17M) +Cells touched = 19452462 (19.5M) +Particle comms = 1043403 (1.04M) +Boundary collides = 1648178 (1.65M) +Boundary exits = 0 (0K) +SurfColl checks = 0 (0K) +SurfColl occurs = 0 (0K) +Surf reactions = 0 (0K) +Collide attempts = 410967 (0.411M) +Collide occurs = 102737 (0.103M) +Reactions = 700 (0.7K) +Particles stuck = 0 +Axisymm bad moves = 0 + +Particle-moves/CPUsec/proc: 6.02483e+06 +Particle-moves/step: 10422.2 +Cell-touches/particle/step: 4.66612 +Particle comm iterations/step: 2.9925 +Particle fraction communicated: 0.250284 +Particle fraction colliding with boundary: 0.395353 +Particle fraction exiting boundary: 0 +Surface-checks/particle/step: 0 +Surface-collisions/particle/step: 0 +Surf-reactions/particle/step: 0 +Collision-attempts/particle/step: 0.0985799 +Collisions/particle/step: 0.0246438 +Reactions/particle/step: 0.000167911 + +Gas reaction tallies: + style qk #-of-reactions 45 + reaction N2 + N2 --> N + N + N2: 672 + reaction N2 + N --> N + N + N: 28 + +Particles: 2675 ave 2725 max 2603 min +Histogram: 1 0 0 0 0 0 2 0 0 1 +Cells: 250 ave 250 max 250 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +GhostCell: 110 ave 110 max 110 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +EmptyCell: 0 ave 0 max 0 min +Histogram: 4 0 0 0 0 0 0 0 0 0 diff --git a/examples/regions/air.species b/examples/regions/air.species new file mode 100644 index 000000000..d102d9070 --- /dev/null +++ b/examples/regions/air.species @@ -0,0 +1,24 @@ +# Species data + +# ID +# Molwt (amu) +# Molmass (kg) +# Rotational dof +# RotRel +# Vibrational dof +# VibRel +# VibTemp (K) +# species wt +# charge + +O2 32.00 5.31E-26 2 0.2 2 5.58659E-5 2256.0 1.0 0.0 +N2 28.016 4.65E-26 2 0.2 2 1.90114E-5 3371.0 1.0 0.0 +O 16.00 2.65E-26 0 0.0 0 0.0 0.0 1.0 0.0 +N 14.008 2.325E-26 0 0.0 0 0.0 0.0 1.0 0.0 +NO 30.008 4.98E-26 2 0.2 2 7.14285E-4 2719.0 1.0 0.0 +O2+ 32.00 5.31E-26 2 0.2 2 5.58659E-5 2256.0 1.0 1.0 +N2+ 28.016 4.65E-26 2 0.2 2 1.90114E-5 3371.0 1.0 1.0 +O+ 16.00 2.65E-26 0 0.0 0 0.0 0.0 1.0 1.0 +N+ 14.008 2.325E-26 0 0.0 0 0.0 0.0 1.0 1.0 +NO+ 30.008 4.98E-26 2 0.2 2 7.14285E-4 2719.0 1.0 1.0 +e 0.001 9.10938188E-31 0 0.0 0 0.0 0.0 1.0 -1.0 diff --git a/examples/regions/air.vss b/examples/regions/air.vss new file mode 100644 index 000000000..42ddebb0d --- /dev/null +++ b/examples/regions/air.vss @@ -0,0 +1,18 @@ +# VSS collision model parameters for each species + +# diameter (m) +# omega +# tref +# alpha + +O2 3.96E-10 0.77 273.15 1.4 +N2 4.07E-10 0.74 273.15 1.6 +O 3.0E-10 0.80 273.15 1.0 +N 3.0E-10 0.80 273.15 1.0 +NO 4.0E-10 0.80 273.15 1.0 +O2+ 3.96E-10 0.77 273.15 1.4 +N2+ 4.07E-10 0.74 273.15 1.6 +O+ 3.0E-10 0.80 273.15 1.0 +N+ 3.0E-10 0.80 273.15 1.0 +NO+ 4.0E-10 0.80 273.15 1.0 +e 7.0E-13 0.50 273.15 1.0 diff --git a/examples/regions/in.regions b/examples/regions/in.regions new file mode 100644 index 000000000..35633d5c1 --- /dev/null +++ b/examples/regions/in.regions @@ -0,0 +1,49 @@ +################################################################################ +# emit through composite regions +# +# Covers region union, intersect, plane and sphere, none of which appear in any +# other deck, and exercises a union nested inside an intersect -- the case the +# KOKKOS flattened-region path handles with an RPN token stream rather than a +# flat primitive list. fix emit/face is region-limited, so the particle count +# is itself the check that the region test agrees between host and device. +# +# Note: +# - The "comm/sort" option to the "global" command is used to match MPI runs. +# - The "twopass" option is used to match Kokkos runs. +################################################################################ + +seed 12345 +dimension 3 +global gridcut 0.0 comm/sort yes +boundary o o o +create_box 0 10 0 10 0 10 +create_grid 8 8 8 +balance_grid rcb cell + +species air.species N O +mixture air N O vstream 100.0 0 0 temp 300.0 +global nrho 1.0e18 fnum 1.0e15 + +# primitives, then a union nested inside an intersect + +# two disjoint slabs whose union leaves a gap in y, intersected with a sphere +# centred on the emit face and a half space. The union is nested inside the +# intersect, so a flat primitive list cannot express it + +region lo block 0 10 0 4 0 10 +region hi block 0 10 6 10 0 10 +region band union 2 lo hi +region ball sphere 0 5 5 6 +region cut plane 0 0 4 0 0 1 +region both intersect 3 band ball cut + +collide vss air air.vss + +fix in emit/face air xlo region both perspecies no twopass + +compute t temp +compute n reduce sum vx +stats 50 +stats_style step np nattempt ncoll c_t c_n +timestep 1.0e-5 +run 200 diff --git a/examples/regions/log.22Aug26.mpi_1.regions b/examples/regions/log.22Aug26.mpi_1.regions new file mode 100644 index 000000000..2c6400ca1 --- /dev/null +++ b/examples/regions/log.22Aug26.mpi_1.regions @@ -0,0 +1,121 @@ +SPARTA (24 Sep 2025) +Running on 1 MPI task(s) +################################################################################ +# emit through composite regions +# +# Covers region union, intersect, plane and sphere, none of which appear in any +# other deck, and exercises a union nested inside an intersect -- the case the +# KOKKOS flattened-region path handles with an RPN token stream rather than a +# flat primitive list. fix emit/face is region-limited, so the particle count +# is itself the check that the region test agrees between host and device. +# +# Note: +# - The "comm/sort" option to the "global" command is used to match MPI runs. +# - The "twopass" option is used to match Kokkos runs. +################################################################################ + +seed 12345 +dimension 3 +global gridcut 0.0 comm/sort yes +boundary o o o +create_box 0 10 0 10 0 10 +Created orthogonal box = (0 0 0) to (10 10 10) +create_grid 8 8 8 +Created 512 child grid cells + CPU time = 0.000855296 secs + create/ghost percent = 86.7195 13.2805 +balance_grid rcb cell +Balance grid migrated 0 cells + CPU time = 0.000161742 secs + reassign/sort/migrate/ghost percent = 60.6521 0.86063 9.87128 28.6159 + +species air.species N O +mixture air N O vstream 100.0 0 0 temp 300.0 +global nrho 1.0e18 fnum 1.0e15 + +# primitives, then a union nested inside an intersect + +# two disjoint slabs whose union leaves a gap in y, intersected with a sphere +# centred on the emit face and a half space. The union is nested inside the +# intersect, so a flat primitive list cannot express it + +region lo block 0 10 0 4 0 10 +region hi block 0 10 6 10 0 10 +region band union 2 lo hi +region ball sphere 0 5 5 6 +region cut plane 0 0 4 0 0 1 +region both intersect 3 band ball cut + +collide vss air air.vss + +fix in emit/face air xlo region both perspecies no twopass + +compute t temp +compute n reduce sum vx +stats 50 +stats_style step np nattempt ncoll c_t c_n +timestep 1.0e-5 +run 200 +Memory usage per proc in Mbytes: + particles (ave,min,max) = 0 0 0 + grid (ave,min,max) = 1.51379 1.51379 1.51379 + surf (ave,min,max) = 0 0 0 + modify (ave,min,max) = 0 0 0 + total (ave,min,max) = 1.51379 1.51379 1.51379 +Step Np Natt Ncoll c_t c_n + 0 0 0 0 0 0 + 50 4891 0 0 428.26388 2704136.7 + 100 9622 3 2 428.03675 5373272.5 + 150 14289 6 3 427.36047 7982785.1 + 200 18729 3 0 425.32081 10468961 +Loop time of 0.0310932 on 1 procs for 200 steps with 18729 particles +Performance: 6432.282 timesteps/s, 120.470 Mparticle-step/s + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Move | 0.014716 | 0.014716 | 0.014716 | 0.0 | 47.33 +Coll | 0.0040274 | 0.0040274 | 0.0040274 | 0.0 | 12.95 +Sort | 0.003542 | 0.003542 | 0.003542 | 0.0 | 11.39 +Comm | 7.855e-05 | 7.855e-05 | 7.855e-05 | 0.0 | 0.25 +Modify | 0.0082092 | 0.0082092 | 0.0082092 | 0.0 | 26.40 +Output | 0.00045179 | 0.00045179 | 0.00045179 | 0.0 | 1.45 +MPI Sync| 5.5186e-05 | 5.5186e-05 | 5.5186e-05 | 0.0 | 0.18 +Other | | 1.296e-05 | | | 0.04 + +Particle moves = 1920940 (1.92M) +Cells touched = 1931536 (1.93M) +Particle comms = 0 (0K) +Boundary collides = 0 (0K) +Boundary exits = 1058 (1.06K) +SurfColl checks = 0 (0K) +SurfColl occurs = 0 (0K) +Surf reactions = 0 (0K) +Collide attempts = 556 (0.556K) +Collide occurs = 384 (0.384K) +Reactions = 0 (0K) +Particles stuck = 0 +Axisymm bad moves = 0 + +Particle-moves/CPUsec/proc: 6.17801e+07 +Particle-moves/step: 9604.7 +Cell-touches/particle/step: 1.00552 +Particle comm iterations/step: 1 +Particle fraction communicated: 0 +Particle fraction colliding with boundary: 0 +Particle fraction exiting boundary: 0.000550772 +Surface-checks/particle/step: 0 +Surface-collisions/particle/step: 0 +Surf-reactions/particle/step: 0 +Collision-attempts/particle/step: 0.000289442 +Collisions/particle/step: 0.000199902 +Reactions/particle/step: 0 + +Particles: 18729 ave 18729 max 18729 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Cells: 512 ave 512 max 512 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +GhostCell: 0 ave 0 max 0 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +EmptyCell: 0 ave 0 max 0 min +Histogram: 1 0 0 0 0 0 0 0 0 0 diff --git a/examples/regions/log.22Aug26.mpi_4.regions b/examples/regions/log.22Aug26.mpi_4.regions new file mode 100644 index 000000000..ab725d4a6 --- /dev/null +++ b/examples/regions/log.22Aug26.mpi_4.regions @@ -0,0 +1,122 @@ +SPARTA (24 Sep 2025) +Running on 4 MPI task(s) +################################################################################ +# emit through composite regions +# +# Covers region union, intersect, plane and sphere, none of which appear in any +# other deck, and exercises a union nested inside an intersect -- the case the +# KOKKOS flattened-region path handles with an RPN token stream rather than a +# flat primitive list. fix emit/face is region-limited, so the particle count +# is itself the check that the region test agrees between host and device. +# +# Note: +# - The "comm/sort" option to the "global" command is used to match MPI runs. +# - The "twopass" option is used to match Kokkos runs. +################################################################################ + +seed 12345 +dimension 3 +global gridcut 0.0 comm/sort yes +boundary o o o +create_box 0 10 0 10 0 10 +Created orthogonal box = (0 0 0) to (10 10 10) +create_grid 8 8 8 +WARNING: Could not acquire nearby ghost cells b/c grid partition is not clumped (/home/user/sparta/src/grid.cpp:486) +Created 512 child grid cells + CPU time = 0.00103162 secs + create/ghost percent = 95.0818 4.91817 +balance_grid rcb cell +Balance grid migrated 384 cells + CPU time = 0.000435365 secs + reassign/sort/migrate/ghost percent = 52.0556 0.443766 21.6731 25.8275 + +species air.species N O +mixture air N O vstream 100.0 0 0 temp 300.0 +global nrho 1.0e18 fnum 1.0e15 + +# primitives, then a union nested inside an intersect + +# two disjoint slabs whose union leaves a gap in y, intersected with a sphere +# centred on the emit face and a half space. The union is nested inside the +# intersect, so a flat primitive list cannot express it + +region lo block 0 10 0 4 0 10 +region hi block 0 10 6 10 0 10 +region band union 2 lo hi +region ball sphere 0 5 5 6 +region cut plane 0 0 4 0 0 1 +region both intersect 3 band ball cut + +collide vss air air.vss + +fix in emit/face air xlo region both perspecies no twopass + +compute t temp +compute n reduce sum vx +stats 50 +stats_style step np nattempt ncoll c_t c_n +timestep 1.0e-5 +run 200 +Memory usage per proc in Mbytes: + particles (ave,min,max) = 0 0 0 + grid (ave,min,max) = 1.51379 1.51379 1.51379 + surf (ave,min,max) = 0 0 0 + modify (ave,min,max) = 0 0 0 + total (ave,min,max) = 1.51379 1.51379 1.51379 +Step Np Natt Ncoll c_t c_n + 0 0 0 0 0 0 + 50 4865 0 0 433.93649 2739350.3 + 100 9606 3 3 432.13116 5381787.5 + 150 14212 5 2 431.30795 7988585 + 200 18767 8 3 428.98491 10506060 +Loop time of 0.0158631 on 4 procs for 200 steps with 18767 particles +Performance: 12607.857 timesteps/s, 236.612 Mparticle-step/s + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Move | 2.044e-05 | 0.0034843 | 0.0069663 | 5.9 | 21.97 +Coll | 6.5659e-05 | 0.0009506 | 0.0018313 | 0.0 | 5.99 +Sort | 1.9298e-05 | 0.00064624 | 0.001284 | 0.0 | 4.07 +Comm | 0.00066779 | 0.00076078 | 0.00081874 | 0.0 | 4.80 +Modify | 1.6977e-05 | 0.0020076 | 0.0040406 | 4.4 | 12.66 +Output | 0.00015516 | 0.00021346 | 0.00025356 | 0.0 | 1.35 +MPI Sync| 0.00067965 | 0.0077919 | 0.014819 | 7.9 | 49.12 +Other | | 8.219e-06 | | | 0.05 + +Particle moves = 1916219 (1.92M) +Cells touched = 1926975 (1.93M) +Particle comms = 37 (0.037K) +Boundary collides = 0 (0K) +Boundary exits = 1029 (1.03K) +SurfColl checks = 0 (0K) +SurfColl occurs = 0 (0K) +Surf reactions = 0 (0K) +Collide attempts = 561 (0.561K) +Collide occurs = 388 (0.388K) +Reactions = 0 (0K) +Particles stuck = 0 +Axisymm bad moves = 0 + +Particle-moves/CPUsec/proc: 3.01993e+07 +Particle-moves/step: 9581.09 +Cell-touches/particle/step: 1.00561 +Particle comm iterations/step: 1.13 +Particle fraction communicated: 1.93089e-05 +Particle fraction colliding with boundary: 0 +Particle fraction exiting boundary: 0.000536995 +Surface-checks/particle/step: 0 +Surface-collisions/particle/step: 0 +Surf-reactions/particle/step: 0 +Collision-attempts/particle/step: 0.000292764 +Collisions/particle/step: 0.000202482 +Reactions/particle/step: 0 + +Particles: 4691.75 ave 9408 max 0 min +Histogram: 2 0 0 0 0 0 0 0 0 2 +Cells: 128 ave 128 max 128 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +GhostCell: 72 ave 72 max 72 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +EmptyCell: 72 ave 72 max 72 min +Histogram: 4 0 0 0 0 0 0 0 0 0 diff --git a/examples/relax_const/in.relax_const b/examples/relax_const/in.relax_const index 2cfc42308..c639ddd8b 100644 --- a/examples/relax_const/in.relax_const +++ b/examples/relax_const/in.relax_const @@ -37,7 +37,23 @@ compute Ttrans reduce ave c_T[1] compute rot grid all all trot compute Trot reduce ave c_rot[1] -stats_style step cpu np nattempt ncoll c_Ttrans c_Trot +# per-grid flux and Sonine moment diagnostics, reduced to scalars for stats + +compute ef eflux/grid all all heatx heaty heatz +compute EF reduce ave c_ef[1] c_ef[3] +compute pf pflux/grid all all momxx momyy momxy +compute PF reduce ave c_pf[1] c_pf[3] +compute sn sonine/grid all all a x 1 b xy 1 +compute SN reduce ave c_sn[1] c_sn[2] + +# ke/particle needs a deck with collisions: without them particle velocities +# never change and any reduction of it is constant for the whole run + +compute kep ke/particle +compute KE reduce max c_kep + +stats_style step cpu np nattempt ncoll c_Ttrans c_Trot & + c_EF[1] c_EF[2] c_PF[1] c_PF[2] c_SN[1] c_SN[2] c_KE timestep 1.00E-9 run 200 diff --git a/examples/relax_const/log.24Aug26.mpi_1.relax_const b/examples/relax_const/log.24Aug26.mpi_1.relax_const new file mode 100644 index 000000000..c076d7fd1 --- /dev/null +++ b/examples/relax_const/log.24Aug26.mpi_1.relax_const @@ -0,0 +1,328 @@ +SPARTA (24 Sep 2025) +Running on 1 MPI task(s) +################################################################################ +# thermal gas in a 3d box with collisions +# particles reflect off global box boundaries +# +# Note: +# - The "comm/sort” option to the “global” command is used to match MPI runs. +# - The “twopass” option is used to match Kokkos runs. +# The "comm/sort" and "twopass" options should not be used for production runs. +################################################################################ + +seed 12345 +dimension 3 +global gridcut 1.0e-5 comm/sort yes + +boundary rr rr rr + +create_box 0 0.0001 0 0.0001 0 0.0001 +Created orthogonal box = (0 0 0) to (0.0001 0.0001 0.0001) +create_grid 3 3 3 +Created 27 child grid cells + CPU time = 0.00155915 secs + create/ghost percent = 97.7371 2.26291 + +balance_grid rcb part +Balance grid migrated 0 cells + CPU time = 0.000108832 secs + reassign/sort/migrate/ghost percent = 72.4575 0.101073 20.5298 6.91157 + +species n2.species N2 +mixture air N2 vstream 0.0 0.0 0.0 temp 10000.0 trot 100.0 + +global nrho 7.07043E22 +global fnum 7.07043E5 + +collide vss air n2.vss relax constant + +create_particles air n 1000000 twopass +Created 1000000 particles + CPU time = 0.236373 secs + +stats 1 +compute temp temp +compute T thermal/grid all all temp +compute Ttrans reduce ave c_T[1] + +compute rot grid all all trot +compute Trot reduce ave c_rot[1] + +# per-grid flux and Sonine moment diagnostics, reduced to scalars for stats + +compute ef eflux/grid all all heatx heaty heatz +compute EF reduce ave c_ef[1] c_ef[3] +compute pf pflux/grid all all momxx momyy momxy +compute PF reduce ave c_pf[1] c_pf[3] +compute sn sonine/grid all all a x 1 b xy 1 +compute SN reduce ave c_sn[1] c_sn[2] + +# ke/particle needs a deck with collisions: without them particle velocities +# never change and any reduction of it is constant for the whole run + +compute kep ke/particle +compute KE reduce max c_kep + +stats_style step cpu np nattempt ncoll c_Ttrans c_Trot c_EF[1] c_EF[2] c_PF[1] c_PF[2] c_SN[1] c_SN[2] c_KE + +timestep 1.00E-9 +run 200 +Memory usage per proc in Mbytes: + particles (ave,min,max) = 96.875 96.875 96.875 + grid (ave,min,max) = 1.51379 1.51379 1.51379 + surf (ave,min,max) = 0 0 0 + modify (ave,min,max) = 0.00926971 0.00926971 0.00926971 + total (ave,min,max) = 98.3981 98.3981 98.3981 +Step CPU Np Natt Ncoll c_Ttrans c_Trot c_EF[1] c_EF[2] c_PF[1] c_PF[2] c_SN[1] c_SN[2] c_KE + 0 0 1000000 0 0 9993.6754 99.967421 -799536 -163856.54 97521.817 -164.18772 -48637266 -1.1911007e+11 2.0981748e-18 + 1 0.36313292 1000000 896601 299278 9538.5717 782.66215 -748830.53 -308145.97 92996.733 -211.36562 -45601986 -1.2696383e+11 2.0043023e-18 + 2 0.73155537 1000000 896615 296049 9140.0641 1380.3477 -365182.88 -109668.56 89126.457 -109.67962 -22203588 -5.6791945e+10 2.4214672e-18 + 3 1.122644 1000000 896612 292699 8787.642 1908.9736 -453606.25 -228467.98 85719.249 -88.297792 -27613911 -2.0253198e+10 2.0043023e-18 + 4 1.4947184 1000000 896614 290224 8477.963 2373.534 -466618.57 -358934.8 82756.236 16.774952 -28361801 7.368534e+10 1.9243524e-18 + 5 1.8167204 1000000 896616 286939 8206.4992 2780.7463 -104945.84 -55003.388 80164.703 106.9905 -6403993.7 6.7659888e+10 1.7556595e-18 + 6 2.1941772 1000000 896624 285071 7967.0237 3139.9572 -228025.58 -50850.175 77792.395 1.139396 -13875233 1.3612137e+10 1.8947772e-18 + 7 2.5728622 1000000 896619 281903 7754.4248 3458.9203 -236209.96 112089.29 75758.189 -24.990502 -14436938 7.3000501e+09 1.6954565e-18 + 8 2.956933 1000000 896623 281074 7566.8259 3740.2512 -169248.8 -115345.17 73916.936 84.71534 -10351391 3.9660281e+10 1.6833302e-18 + 9 3.2784356 1000000 896625 278910 7399.7478 3990.825 -284761.86 -231878.61 72331.665 76.108335 -17349569 5.9457493e+09 1.5998553e-18 + 10 3.6198268 1000000 896628 277217 7252.3246 4212.0019 -130987.25 5664.1997 70898.446 126.89502 -8005013.3 7.7127931e+10 1.5998553e-18 + 11 4.0009562 1000000 896621 276372 7120.928 4409.132 -413808.65 131892.3 69522.798 141.29624 -25157272 7.613193e+10 1.5998553e-18 + 12 4.3888884 1000000 896627 274766 7006.8109 4580.3794 -344432.59 -179048.46 68299.503 93.46393 -20927670 3.3232098e+10 1.6012636e-18 + 13 4.7732916 1000000 896624 274110 6906.7881 4730.359 -81207.492 -273806.41 67442.814 102.9071 -4918109.3 4.6040182e+10 1.6012636e-18 + 14 5.1503943 1000000 896633 273220 6814.3584 4869.0295 -252941.77 -187198.68 66521.582 38.359027 -15320375 2.3808734e+10 1.4711785e-18 + 15 5.5529033 1000000 896629 271455 6732.4045 4991.9654 -279921.85 97911.808 65590.536 -69.238285 -16984983 -1.098619e+10 1.5404075e-18 + 16 5.9341932 1000000 896623 271411 6660.129 5100.3772 -180383.1 65211.85 64939.711 -94.806455 -10909551 -2.9856003e+10 1.4455897e-18 + 17 6.3256485 1000000 896630 270794 6595.4736 5197.3747 -302471.5 42210.465 64386.835 -68.327844 -18329730 2.6527026e+09 1.4319077e-18 + 18 6.7201599 1000000 896639 269343 6535.5527 5287.2326 -101651.52 -140459.98 63909.471 -43.07326 -6044031.7 8.6822181e+09 1.4319077e-18 + 19 7.1168066 1000000 896628 269802 6481.5301 5368.2252 55471.512 -93900.61 63408.37 -132.93977 3457273.4 -4.3807315e+10 1.4319077e-18 + 20 7.5077803 1000000 896635 269152 6433.2254 5440.6354 -14552.093 -210961.96 62945.468 -112.88632 -796437.77 -5.0022219e+10 1.4228999e-18 + 21 7.9022497 1000000 896631 268075 6391.6337 5503.0779 -131362.29 -309825.15 62571.716 -111.45362 -7916977.7 -4.3876386e+10 1.4228999e-18 + 22 8.2946419 1000000 896632 268029 6354.1074 5559.3542 -163977.09 -146831.72 62234.223 -96.871147 -9927821.8 -5.4496686e+10 1.3054883e-18 + 23 8.7025024 1000000 896632 266603 6321.4689 5608.3348 -110204.66 -255154.58 61854.34 -91.21344 -6706776 -6.6969506e+10 1.6586047e-18 + 24 9.0887237 1000000 896624 267604 6289.7779 5655.9106 -11829.203 -313824.93 61487.732 -7.6554192 -678821.84 -9.6254047e+09 1.5256298e-18 + 25 9.4765074 1000000 896634 266732 6262.5978 5696.7075 81953.647 -217909.52 61186.042 75.275621 5039007.6 2.3312471e+10 1.2784553e-18 + 26 9.8772074 1000000 896632 266822 6240.7928 5729.4809 102853.53 -221191.65 60991.913 92.291036 6274094.6 4.7170871e+10 1.2471991e-18 + 27 10.257019 1000000 896628 265628 6219.795 5761.0165 31139.774 -102302.31 60793.483 124.55303 1896817.1 7.2958213e+10 1.4142148e-18 + 28 10.690311 1000000 896628 266032 6203.0771 5786.1124 -26139.901 -117705.18 60607.223 104.85706 -1565154.4 3.9896238e+10 1.2323831e-18 + 29 11.112106 1000000 896631 266507 6185.3501 5812.6937 27376.231 -177400.04 60405.372 97.230173 1656576.8 5.475481e+10 1.4217649e-18 + 30 11.56087 1000000 896634 265090 6170.23 5835.3674 132359.96 -139749.67 60263.812 40.114711 8036009.6 1.2477645e+10 1.3825327e-18 + 31 11.98322 1000000 896623 265006 6156.474 5855.9762 194940.51 -259197.81 60039.893 22.88623 11871659 1.3113605e+10 1.3825327e-18 + 32 12.380136 1000000 896630 265511 6144.9018 5873.2528 56900.145 -164444.06 59932.091 1.3143697 3481295.8 5.3130055e+09 1.3825327e-18 + 33 12.712777 1000000 896634 265471 6132.6622 5891.5994 -9728.5792 -21486.316 59816.903 19.536942 -561468.86 2.0698635e+10 1.3825327e-18 + 34 13.124601 1000000 896629 265180 6123.3055 5905.6299 -112460.48 -72578.315 59705.564 11.932772 -6817797.2 1.8999096e+10 1.399603e-18 + 35 13.548339 1000000 896631 264495 6115.7172 5916.994 -76100.631 -985.11357 59597.463 -52.40945 -4602892.5 -2.5036713e+10 1.33823e-18 + 36 13.97525 1000000 896630 264781 6105.3836 5932.4735 -123471.35 64214.665 59517.153 -84.383858 -7506103.7 -3.3851063e+10 1.3692234e-18 + 37 14.397748 1000000 896633 265028 6097.187 5944.8037 -99527.402 -53042.619 59536.432 -49.09537 -6018871.5 -7.8190434e+09 1.425751e-18 + 38 14.814224 1000000 896635 264791 6092.3023 5952.1224 -13188.227 -182665.27 59497.25 -26.113938 -786786.72 4.1289876e+09 1.2324427e-18 + 39 15.198263 1000000 896638 264653 6086.2753 5961.1663 76535.628 29338.581 59384.311 9.5588657 4671750 1.3484905e+10 1.3064472e-18 + 40 15.6142 1000000 896633 265256 6076.9351 5975.146 97687.584 28589.994 59399.601 14.334016 5922169.8 1.2422732e+10 1.5811073e-18 + 41 16.026238 1000000 896634 265479 6074.9645 5978.1406 170186.36 94578.98 59349.582 37.273604 10357123 1.5298392e+10 1.5811073e-18 + 42 16.470765 1000000 896629 264925 6070.8354 5984.383 19598.461 -16658.848 59252.429 10.234405 1177445.8 -5.3175046e+09 1.5811073e-18 + 43 16.877442 1000000 896628 264863 6068.1247 5988.4635 98842.856 -272560.91 59244.673 -56.581616 6047144 -2.9777017e+10 1.1986905e-18 + 44 17.297963 1000000 896629 263930 6068.1027 5988.4469 34873.113 -156592.19 59242.197 16.370685 2145663 -1.2262601e+09 1.1880619e-18 + 45 17.712519 1000000 896630 264917 6065.2865 5992.6593 100205.44 -119607.35 59184.529 -34.286727 6110658.4 -2.4984291e+10 1.4162411e-18 + 46 18.13822 1000000 896635 264448 6060.9999 5999.0984 69733.532 -112887.63 59149.203 -62.242487 4205135 -2.8404118e+10 1.4162411e-18 + 47 18.558553 1000000 896636 265002 6059.6249 6001.1471 -3345.7317 -55544.969 59176.357 -1.163299 -238796.85 -1.8480729e+10 1.4162411e-18 + 48 18.977814 1000000 896641 264267 6057.4346 6004.5156 -205391.75 -89566.745 59108.899 32.175941 -12536135 8.2334212e+08 1.1817763e-18 + 49 19.40122 1000000 896644 264044 6056.5468 6005.8437 -36440.058 41594.44 59093.585 13.653846 -2245897.3 -1.9896747e+10 1.1817763e-18 + 50 19.826645 1000000 896633 264391 6052.9285 6011.2886 -63388.049 126231.41 59040.991 20.414128 -3874100.2 -5.6844041e+09 1.1293103e-18 + 51 20.255294 1000000 896642 264450 6052.8694 6011.3839 -64164.03 160327.57 59043.281 27.619013 -3916758.3 8.0196479e+09 1.337444e-18 + 52 20.684317 1000000 896631 264315 6052.3463 6012.1489 -98572.104 48066.464 59096.414 10.4559 -6004318.1 1.3386072e+10 1.5215844e-18 + 53 21.112299 1000000 896636 264460 6050.0987 6015.5336 -95189.021 240637.03 59119.416 -21.66877 -5793663.2 3.0349784e+09 1.1718429e-18 + 54 21.538374 1000000 896629 264171 6046.8118 6020.486 -1795.5725 304861.76 59146.112 -117.22768 -111432.97 -2.8201931e+10 1.1916798e-18 + 55 21.959125 1000000 896631 263968 6046.3227 6021.1897 71906.771 187185.97 59106.228 -36.689858 4381226.3 -1.8426357e+10 1.5841032e-18 + 56 22.399376 1000000 896633 264377 6046.3364 6021.2184 22353.471 197202.04 59063.004 9.5530218 1340300.3 7.4476305e+08 1.4623201e-18 + 57 22.830146 1000000 896625 263644 6045.4023 6022.5835 -136649.7 242977.37 59114.58 87.022409 -8373891.4 3.7666948e+10 1.4623201e-18 + 58 23.251303 1000000 896628 264362 6043.3195 6025.6985 -93764.982 186025.52 59039.059 11.672309 -5700214.2 -2.7960609e+09 1.4623201e-18 + 59 23.688328 1000000 896625 263981 6044.5623 6023.8402 -83676.262 85047.55 59057.328 -11.401895 -5109774.2 -2.6705042e+10 1.1871761e-18 + 60 24.117068 1000000 896632 264335 6044.4455 6023.9753 -157271.32 64391.875 59046.057 25.94261 -9574115.4 7.6805257e+08 1.3719682e-18 + 61 24.559113 1000000 896630 263675 6045.7177 6021.9928 -110470.81 132640.59 59147.875 94.142012 -6732101.7 1.9083269e+10 1.3719682e-18 + 62 25.007159 1000000 896635 264397 6045.3947 6022.5008 -83038.747 -32517.578 59068.911 128.54988 -5089947.6 4.5395419e+10 1.6373716e-18 + 63 25.452347 1000000 896631 264747 6045.9047 6021.7445 -74903.269 46684.749 59097.634 29.541531 -4586127.3 6.174521e+09 1.303665e-18 + 64 25.905273 1000000 896626 264459 6042.5773 6026.7577 -144077.03 -79589.364 59117.843 90.642286 -8785574.8 1.2304519e+10 1.2901936e-18 + 65 26.370063 1000000 896624 263762 6041.6717 6028.1223 38576.432 -20770.41 59055.471 64.041087 2375921.5 -1.1796062e+10 1.2901936e-18 + 66 26.806807 1000000 896629 264315 6038.914 6032.1903 105199.66 -10528.045 58942.689 83.893741 6406761.6 2.5770665e+10 1.2338196e-18 + 67 27.25795 1000000 896629 264204 6037.0781 6034.9692 79655.052 -63140.249 58903.221 13.718549 4840368.6 1.2364653e+10 1.2338196e-18 + 68 27.695399 1000000 896624 264523 6038.2853 6033.1515 132976.29 -165456.7 58939.855 29.138269 8082593.2 2.0928893e+10 1.2887756e-18 + 69 28.13111 1000000 896623 264869 6037.7884 6033.9343 64509.185 -25333.434 58903.396 33.690019 3940939.8 1.7004649e+10 1.2604533e-18 + 70 28.572374 1000000 896629 264396 6036.3953 6036.0335 22422.859 -173725.21 58965.133 -21.701708 1410745.2 -5.0523505e+09 1.3011535e-18 + 71 29.007529 1000000 896634 264654 6035.7729 6036.9627 -48075.208 -97519.561 58962.603 -2.8022174 -2883004.8 -1.3354898e+10 1.4306804e-18 + 72 29.434681 1000000 896636 264645 6032.8446 6041.3016 66831.817 -73358.789 58977.73 32.518958 4088148.2 5.226342e+09 1.4306804e-18 + 73 29.87603 1000000 896626 265294 6033.6302 6040.1237 67555.552 -190449.28 58990.804 -31.479127 4075375.2 -2.7048386e+10 1.347782e-18 + 74 30.313746 1000000 896623 264213 6033.9334 6039.6074 26209.058 -46249.867 58966.251 -37.308736 1596248.5 -1.0504254e+10 1.347782e-18 + 75 30.74493 1000000 896630 263972 6033.0589 6040.9523 34923.471 5108.5747 58944.813 69.065742 2101289.4 4.5621668e+10 1.347782e-18 + 76 31.170368 1000000 896625 264025 6034.2695 6039.1768 32217.665 -33580.767 58969.847 16.632481 1938061.6 1.7280625e+10 1.2810004e-18 + 77 31.600651 1000000 896631 264527 6037.5833 6034.1963 -1277.4173 28268.7 58940.676 -21.707546 -46862.854 5.1616992e+09 1.4550013e-18 + 78 32.029568 1000000 896627 263988 6037.1382 6034.8997 10852.3 155735.12 58910.974 16.698126 681168.28 1.7887894e+10 1.4174091e-18 + 79 32.453847 1000000 896631 263980 6037.4275 6034.4358 197056.68 194184.23 58881.628 -54.575566 12020554 -7.6510844e+09 1.4174091e-18 + 80 32.899405 1000000 896624 264207 6035.2791 6037.6154 233109.88 95823.228 58899.423 -25.905349 14220334 1.8557031e+10 1.4699066e-18 + 81 33.33234 1000000 896626 264453 6036.6341 6035.5926 170450.63 57269.353 58923.586 -7.9909958 10401573 2.3374945e+10 1.2745165e-18 + 82 33.756735 1000000 896628 263810 6036.3228 6036.1043 -4042.6118 76092.044 58914.186 42.345996 -207590.94 3.6177375e+10 1.2745165e-18 + 83 34.193692 1000000 896632 264105 6035.6885 6037.0973 96963.478 -46772.086 58909.061 28.544204 5959111.9 2.7979152e+10 1.2005194e-18 + 84 34.646248 1000000 896624 264832 6034.513 6038.8848 -92940.883 -127708.37 58917.41 -5.0377965 -5612814.8 9.6865372e+09 1.2556292e-18 + 85 35.090665 1000000 896626 263465 6034.7487 6038.4915 -169361.12 -79897.195 58955.915 35.444286 -10263834 2.0836289e+10 1.2027692e-18 + 86 35.531535 1000000 896628 264666 6037.9902 6033.5823 -129198.13 78683.347 59030.171 12.011969 -7824690.3 1.1654522e+10 1.4994725e-18 + 87 35.979735 1000000 896625 263491 6037.9002 6033.7213 -176955.29 -24582.29 58974.996 -75.399067 -10789089 -1.0676175e+10 1.2595646e-18 + 88 36.424877 1000000 896632 264040 6038.0082 6033.4529 -73140.067 -55496.881 59071.981 -105.34676 -4423148.8 -2.3839335e+10 1.2396872e-18 + 89 36.868967 1000000 896631 264441 6037.5522 6034.1308 -119512.84 -153293.28 58952.73 4.550249 -7219603.5 1.7080728e+10 1.2212721e-18 + 90 37.298202 1000000 896631 264208 6038.2444 6033.099 -66588.133 -181412.88 59057.535 -51.028661 -4008960.7 3.9262707e+09 1.4317994e-18 + 91 37.722652 1000000 896632 264185 6038.7042 6032.3985 -22631.055 -56928.997 59084.188 -34.533919 -1331152.5 7.0041703e+09 1.4317994e-18 + 92 38.154606 1000000 896632 264551 6038.8101 6032.333 38176.183 -11672.622 59067.129 64.923226 2276490.5 5.4160573e+10 1.4452013e-18 + 93 38.589353 1000000 896624 264396 6039.7749 6030.9273 -15878.613 -223457.57 59122.354 93.088817 -1009247.3 5.6189667e+10 1.4452013e-18 + 94 39.027692 1000000 896630 263507 6038.4288 6032.9501 -47315.846 -115828.67 59040.612 61.6029 -2858452.4 4.3947551e+10 1.1685669e-18 + 95 39.44966 1000000 896629 263744 6039.9161 6030.7381 -126292.98 -16584.623 58953.088 73.433258 -7670920.4 3.4384974e+10 1.1738798e-18 + 96 39.901252 1000000 896627 264864 6038.5325 6032.7761 -111258.7 -157111.62 58959.044 -33.177084 -6765312.1 -6.7405161e+09 1.2988227e-18 + 97 40.349774 1000000 896620 264145 6037.558 6034.283 -155901.63 -238908.2 58869.929 -6.381152 -9432528 -9.5705461e+09 1.4110928e-18 + 98 40.800686 1000000 896632 263713 6037.7705 6033.9234 -115306.72 -153114.98 58966.937 4.8023816 -6977110.2 7.7717546e+09 1.4110928e-18 + 99 41.250684 1000000 896632 263626 6040.1783 6030.2759 76982.011 -93600.107 58966.985 54.297144 4677363.7 1.2872967e+10 1.219005e-18 + 100 41.70213 1000000 896630 264403 6039.4363 6031.4394 -74660.872 -5628.2144 58942.836 80.68099 -4550731.1 4.0240314e+10 1.219005e-18 + 101 42.228125 1000000 896625 263738 6042.711 6026.4859 -55235.642 -27932.92 59008.482 62.671457 -3417963.2 3.0867309e+10 1.219005e-18 + 102 42.677538 1000000 896633 264296 6040.8874 6029.2437 13274.475 90893.233 59002.436 81.101818 779807.25 5.2000776e+10 1.1729254e-18 + 103 43.111569 1000000 896637 264190 6038.461 6032.8834 221723.31 14692.397 58969.233 94.691519 13479324 3.3014428e+10 1.6099309e-18 + 104 43.53351 1000000 896638 264570 6036.978 6035.1486 206699.9 -98353.148 58984.668 106.01724 12543736 3.5514272e+10 1.5518565e-18 + 105 43.983184 1000000 896641 264039 6037.3115 6034.6998 45840.806 -140894.64 59033.628 98.145459 2778865.4 6.0290228e+10 1.3391192e-18 + 106 44.42071 1000000 896636 264179 6035.8528 6036.9153 91092.27 -149002.94 58971.266 48.248876 5540160.3 2.7768531e+10 1.3391192e-18 + 107 44.852836 1000000 896633 264604 6036.6995 6035.6548 146542.07 -68586.273 58969.555 3.1416656 8871322.9 6.4088019e+09 1.3391192e-18 + 108 45.272576 1000000 896646 264191 6039.6959 6031.0836 110287.2 -89651.016 59037.307 0.19020075 6642004.7 1.2142137e+10 1.338021e-18 + 109 45.723684 1000000 896637 264288 6040.8063 6029.4108 267093.2 -22299.711 59068.687 -25.969295 16189603 -5.8109509e+09 1.2749039e-18 + 110 46.175087 1000000 896633 264685 6040.944 6029.2337 163408.97 62036.092 59070.609 16.594868 9883676.7 3.9174386e+09 1.2597261e-18 + 111 46.63451 1000000 896631 264377 6040.6721 6029.5927 52675.481 23789.182 59163.783 11.607835 3159087.7 1.8792128e+09 1.2154358e-18 + 112 47.07675 1000000 896629 264580 6036.849 6035.3305 163998.19 -70791.478 59061.695 34.467928 9959411.1 1.2280033e+10 1.2218195e-18 + 113 47.523196 1000000 896627 264388 6036.6402 6035.5858 203335.23 -92161.186 58990.623 14.482134 12388717 1.4342712e+10 1.2589846e-18 + 114 47.969323 1000000 896632 264534 6034.6281 6038.6148 -26246.741 -128016.2 58948.459 87.343564 -1594664.4 4.5671935e+10 1.3689819e-18 + 115 48.410576 1000000 896641 264254 6036.0252 6036.5062 2015.8626 -155867.54 58962.969 51.236126 118796.62 2.302999e+10 1.3689819e-18 + 116 48.845415 1000000 896632 264035 6034.7578 6038.4018 96587.509 -17302.385 58869.634 31.147958 5884487.8 -7.9004343e+09 1.3689819e-18 + 117 49.284392 1000000 896636 263360 6035.0445 6037.9111 10912.417 14105.189 58869.66 22.394517 692389.24 -11083978 1.4635272e-18 + 118 49.716135 1000000 896634 263602 6035.2349 6037.6822 90564.082 -42536.54 58880.975 21.115934 5469869.3 1.349979e+10 1.4635272e-18 + 119 50.092404 1000000 896634 263908 6034.2093 6039.2439 88.96311 -53146.975 58881.626 -95.71415 -24484.916 -3.8123119e+10 1.4635272e-18 + 120 50.506993 1000000 896647 263913 6034.1813 6039.2012 -37158.992 -80113.33 58940.702 -150.74294 -2276244.9 -6.4371259e+10 1.4635272e-18 + 121 50.914115 1000000 896635 264348 6035.7321 6036.9425 -97376.865 -167453.32 58931.884 -73.619399 -5930171.9 -3.8590198e+10 1.3730525e-18 + 122 51.314948 1000000 896633 264363 6035.0106 6038.0265 -67436.67 -55377.463 59041.604 -74.63337 -4037503.2 -4.3959771e+10 1.1680235e-18 + 123 51.716197 1000000 896641 263713 6035.7473 6036.888 -214807.9 -60177.385 58967.741 12.316389 -12983999 -3.3408635e+09 1.2422493e-18 + 124 52.130803 1000000 896635 264685 6034.5311 6038.7108 -126808.47 -44835.734 58936.553 -23.535594 -7613248.4 -6.1312409e+09 1.365193e-18 + 125 52.548227 1000000 896630 263984 6035.1731 6037.7447 -116136.35 13227.326 58865.01 46.52162 -6957800.7 1.7627003e+10 1.365193e-18 + 126 52.97075 1000000 896629 264058 6036.1583 6036.277 64486.682 51269.084 58877.773 -5.5746743 3951738.2 -4.9429774e+09 1.3114428e-18 + 127 53.391391 1000000 896634 263856 6035.6423 6037.0378 103298.79 120965.39 58951.926 37.221747 6308594 2.2590546e+10 1.3114428e-18 + 128 53.81364 1000000 896636 263994 6037.1489 6034.8074 135764.1 -19044.864 58968.865 -26.052256 8301856.9 1.4992097e+10 1.3552717e-18 + 129 54.235426 1000000 896638 264677 6038.0692 6033.4757 156156.5 -113171.43 58913.96 -63.697833 9568015.8 -1.6331798e+10 1.3552717e-18 + 130 54.656628 1000000 896630 264249 6038.5042 6032.8651 113515.16 4855.6964 58935.154 -69.8357 6970864.2 -2.0411697e+10 1.4922938e-18 + 131 55.076346 1000000 896633 264048 6038.8457 6032.2367 -36589.942 -54359.502 58951.177 -48.642253 -2165280 -2.3760082e+10 1.4922938e-18 + 132 55.498333 1000000 896631 265008 6037.3345 6034.5496 -92996.225 -119968.52 58879.062 -91.550982 -5576987.4 -3.1621895e+10 1.4502368e-18 + 133 55.922631 1000000 896632 263778 6038.2683 6033.2036 -25230.672 -138177.21 58931.622 -26.682944 -1479730.4 -6.1918805e+09 1.3841255e-18 + 134 56.346354 1000000 896635 264337 6037.6989 6033.9905 5594.5548 -118177.87 58919.137 42.698533 404363.83 2.8276496e+10 1.3841255e-18 + 135 56.773642 1000000 896633 265165 6035.4707 6037.2654 -31831.59 80235.063 58944.896 -0.48413617 -1899507.4 6.1770758e+09 1.2679854e-18 + 136 57.193322 1000000 896637 263474 6039.0126 6031.973 -46933.886 159212.06 58966.243 44.809668 -2874862.6 1.98148e+10 1.2858805e-18 + 137 57.61268 1000000 896634 264964 6037.4413 6034.3438 -53358.225 89697.629 58907.826 -8.9108006 -3238030.1 4.6356799e+09 1.2106167e-18 + 138 58.028753 1000000 896641 263809 6036.4207 6035.8617 -114389.04 124834.68 58903.232 -46.316884 -6959286.6 -2.1317769e+10 1.2106167e-18 + 139 58.449651 1000000 896629 263938 6035.1714 6037.6908 -55557.052 14804.442 58945.288 -49.020532 -3392157.9 -1.8574321e+10 1.2106167e-18 + 140 58.867989 1000000 896628 263718 6033.1511 6040.7355 -163000.31 -37566.398 58903.768 -54.182979 -9908833.8 -3.2121162e+10 1.2867965e-18 + 141 59.284894 1000000 896637 264539 6032.1059 6042.2938 -176859.76 -105975.56 58944.324 -73.462323 -10761247 -2.4304202e+10 1.2372694e-18 + 142 59.708675 1000000 896629 263773 6032.1359 6042.3084 -33645.219 -194388.29 58942.5 -44.059168 -2027884 -3.536105e+10 1.2106167e-18 + 143 60.130333 1000000 896631 264468 6030.2652 6045.1186 -310580.12 -25557.959 58937.239 0.46716335 -18866192 -1.4599903e+10 1.2646644e-18 + 144 60.552269 1000000 896638 263603 6033.1083 6040.8824 -278710.58 30809.133 58980.354 55.143738 -16928110 1.2513228e+10 1.3077462e-18 + 145 60.973073 1000000 896632 264106 6035.522 6037.2272 -279977.55 186432.83 58927.958 15.514518 -17025770 -9.3167164e+08 1.2646644e-18 + 146 61.393383 1000000 896631 264287 6034.1101 6039.3273 -35218.918 131065.2 58897.352 65.194794 -2148735.2 1.9830033e+10 1.3552741e-18 + 147 61.812221 1000000 896634 263988 6032.2951 6042.1072 -46680.254 104715.41 58896.008 118.10354 -2873843.3 5.4581115e+10 1.2819235e-18 + 148 62.234793 1000000 896630 264071 6031.5949 6043.2204 -42536.611 95910.199 58938.495 152.58694 -2601935.5 7.1816737e+10 1.2357309e-18 + 149 62.659872 1000000 896635 264497 6032.1881 6042.3526 -14984.196 -66116.499 58934.292 152.43075 -899674.17 5.6492166e+10 1.2621793e-18 + 150 63.078899 1000000 896635 264909 6029.2721 6046.7392 110945.88 -9944.6573 58905.078 138.1128 6734879.8 5.8878163e+10 1.3101732e-18 + 151 63.50175 1000000 896636 263807 6030.2002 6045.3043 90078.839 12673.672 58887.337 136.48165 5511621.8 5.0875215e+10 1.3101732e-18 + 152 63.923249 1000000 896634 264255 6029.861 6045.8174 173618.41 -48496.863 58917.109 69.048017 10639453 3.1443314e+10 1.2346603e-18 + 153 64.347134 1000000 896639 264814 6029.9339 6045.7472 137220.89 78701.559 58906.986 -51.952869 8368369.7 -1.5366377e+10 1.2147498e-18 + 154 64.774453 1000000 896643 263974 6028.6734 6047.6152 137471.73 227099.68 58872.914 -21.967931 8419266.9 -1.2268141e+10 1.2605788e-18 + 155 65.199623 1000000 896640 264421 6029.8209 6045.889 -10825.551 194269.1 58903.697 -33.242734 -642661.92 -7.0588312e+08 1.2605788e-18 + 156 65.637073 1000000 896639 264614 6031.7921 6043.0213 -133667.92 5558.4832 58950.613 -58.456132 -8150919.1 -5.9294389e+09 1.4123977e-18 + 157 66.056347 1000000 896644 264351 6032.4424 6042.0032 -20411.758 624.63261 58932.094 7.1301369 -1251056.3 9.8256845e+09 1.4123977e-18 + 158 66.481594 1000000 896640 264192 6032.6786 6041.6263 -40357.311 -51998.781 58888.618 -35.35883 -2448912.4 -5.8789528e+09 1.2147439e-18 + 159 66.903309 1000000 896632 264116 6033.8562 6039.834 -136529.69 149172.28 59011.461 -78.868886 -8282273.6 -3.3587613e+10 1.2096735e-18 + 160 67.327546 1000000 896640 264564 6032.9906 6041.083 -18947.619 14220.119 58951.093 -18.494229 -1167504.4 -2.7646104e+10 1.3735034e-18 + 161 67.749006 1000000 896638 263941 6034.6714 6038.6093 -62698.485 -76947.651 58912.508 -39.270201 -3845933.2 -3.2535565e+10 1.3735034e-18 + 162 68.171637 1000000 896643 263543 6033.6151 6040.1296 -153535.47 76682.674 58931.561 31.796436 -9339708.8 -1.0152287e+10 1.3735034e-18 + 163 68.594178 1000000 896631 263660 6034.2167 6039.2037 -80514.387 207574.06 59024.683 32.500507 -4861553.3 -1.1123867e+10 1.2901004e-18 + 164 69.017376 1000000 896637 264797 6034.0615 6039.442 92124.095 295881.58 58988.891 51.505548 5641802 1.0701302e+10 1.2901004e-18 + 165 69.440956 1000000 896631 264435 6033.7066 6039.9767 114108.17 229274.78 58893.333 43.958114 6956109.5 1.5332199e+09 1.334837e-18 + 166 69.847169 1000000 896632 264242 6031.775 6042.887 161765.32 175881.29 58868.195 15.679289 9814221.6 -2.7125813e+08 1.2907003e-18 + 167 70.247702 1000000 896625 263810 6031.8024 6042.87 3907.367 2647.7524 58848.248 -51.95652 191721.45 -4.7639921e+10 1.1624744e-18 + 168 70.645517 1000000 896631 264587 6029.082 6046.9283 -170315.32 46075.92 58892.943 -43.336561 -10433903 -3.4270421e+10 1.2553257e-18 + 169 71.039712 1000000 896639 263079 6029.5921 6046.1772 -52029.126 -3793.8365 58864.649 -81.468592 -3216552 -3.7100986e+10 1.3583545e-18 + 170 71.435418 1000000 896641 264137 6031.1966 6043.7195 -73600.96 -89997.363 58898.257 -11.039908 -4515855.6 -1.1473721e+10 1.2855286e-18 + 171 71.837953 1000000 896633 263238 6029.1373 6046.8728 5144.017 43588.09 58953.094 40.69424 241934.39 2.0013802e+10 1.1788199e-18 + 172 72.22156 1000000 896637 264999 6029.0023 6047.0201 184935.58 15328.028 58906.892 -10.408797 11183634 5.7746595e+09 1.1788957e-18 + 173 72.62943 1000000 896641 263875 6030.7289 6044.3898 174578.42 -11535.358 58884.318 -91.197228 10584551 -2.5131801e+10 1.2170103e-18 + 174 73.047932 1000000 896634 263869 6032.734 6041.3651 115702.84 -86879.253 58900.574 -92.776243 6976720.7 -1.9637107e+10 1.199193e-18 + 175 73.460079 1000000 896636 264871 6031.9882 6042.4716 82195.284 -12011.687 58825.857 -44.40522 4986760.1 -2.0073574e+10 1.2134014e-18 + 176 73.888765 1000000 896633 265114 6034.3028 6038.9785 245281.52 46737.824 58850.142 -90.67105 14941106 -3.7576559e+10 1.2109379e-18 + 177 74.30125 1000000 896630 264741 6035.6034 6037.0618 323771.64 111886.28 58846.279 -27.880777 19720218 -1.3310395e+10 1.2967739e-18 + 178 74.696528 1000000 896631 264310 6036.5601 6035.6338 236900.42 150842.72 58878.082 6.7840409 14415085 2.9142698e+09 1.1931822e-18 + 179 75.101372 1000000 896627 264343 6037.8025 6033.8007 196098.97 104697.04 59002.704 -17.877424 11947814 -2.0070941e+10 1.3689441e-18 + 180 75.510396 1000000 896625 264404 6037.8069 6033.8323 81355.558 114477.81 59013.13 20.383458 4961986.1 5.7322143e+09 1.4809642e-18 + 181 75.927637 1000000 896621 263914 6037.0158 6035.0722 240925.92 -101031.89 58951.474 28.820653 14648012 1.3761619e+09 1.4809642e-18 + 182 76.281579 1000000 896625 264390 6034.8939 6038.2612 -844.74678 -9923.0521 58867.483 47.70598 -50953.675 2.0388903e+10 1.4809642e-18 + 183 76.62465 1000000 896622 264545 6033.5749 6040.2134 1979.1746 -73541.128 58831.144 -22.537133 127326.77 -6.8801869e+09 1.3299029e-18 + 184 77.012995 1000000 896623 263721 6033.4938 6040.3122 34696.637 -205874.27 58921.294 57.335752 2126171.9 2.3598996e+10 1.1677701e-18 + 185 77.31931 1000000 896623 264839 6032.9583 6041.0749 -6040.4452 -318174.3 58942.447 70.087453 -347967.55 3.4704799e+10 1.4022808e-18 + 186 77.666612 1000000 896626 264014 6032.3861 6041.9596 -24364.911 -140123.16 58956.425 68.917969 -1468894.2 4.8283395e+10 1.4022808e-18 + 187 78.060853 1000000 896626 264108 6032.7285 6041.4382 -120153.98 -167027.67 58977.69 69.755378 -7351787.8 4.8139313e+10 1.3132252e-18 + 188 78.403005 1000000 896626 264048 6031.7962 6042.8434 -111743.33 -65167.148 58870.202 40.72272 -6852716.7 2.5500072e+10 1.1759758e-18 + 189 78.796847 1000000 896624 264414 6031.3765 6043.5351 23709.245 111053.63 58822.828 37.759185 1410195.2 3.1867184e+10 1.3551801e-18 + 190 79.163815 1000000 896622 263728 6032.4184 6041.9293 86575.843 35325.086 58834.579 37.007676 5276966.7 2.5326807e+09 1.2671707e-18 + 191 79.521781 1000000 896625 263884 6030.6147 6044.6439 118620.17 14414.02 58818.142 47.529733 7222356.1 1.7856578e+09 1.2535158e-18 + 192 79.854964 1000000 896616 264083 6028.7378 6047.4199 20233.07 56622.566 58826.24 -3.3188662 1197816.8 -6.5763647e+09 1.3037214e-18 + 193 80.233754 1000000 896626 263432 6031.1503 6043.7818 -192694.98 -69771.2 58820.755 45.360203 -11727187 1.3827161e+09 1.3037214e-18 + 194 80.628123 1000000 896633 264089 6030.8367 6044.2155 -69626.793 -131430.18 58853.329 -27.656646 -4179824.9 -1.6107524e+10 1.2635644e-18 + 195 80.976925 1000000 896631 263844 6033.8962 6039.6561 -152123.41 -59994.868 58880.553 -26.641509 -9216876.3 -6.8198187e+09 1.5190392e-18 + 196 81.371497 1000000 896633 264217 6033.0805 6040.9377 -173895.48 24117.792 58873.519 -6.4128888 -10543111 -5.2052423e+09 1.5190392e-18 + 197 81.723129 1000000 896630 264378 6034.1778 6039.2831 -261941.91 -20092.661 58884.456 -0.78714972 -15980915 5.1701675e+09 1.5190392e-18 + 198 82.032468 1000000 896626 264132 6033.1569 6040.8276 -285344.09 -53259.06 58967.615 -42.163599 -17366211 -1.1042077e+10 1.5190392e-18 + 199 82.348393 1000000 896631 263849 6034.0759 6039.4859 -280402.13 -134453.79 59010.061 15.257374 -17061912 1.460775e+10 1.5190392e-18 + 200 82.696054 1000000 896628 263376 6032.9279 6041.2034 -155551.4 -118304.05 58984.291 -47.23864 -9465120.4 -2.8893098e+10 1.387328e-18 +Loop time of 82.6961 on 1 procs for 200 steps with 1000000 particles +Performance: 2.418 timesteps/s, 2.418 Mparticle-step/s + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Move | 3.3057 | 3.3057 | 3.3057 | 0.0 | 4.00 +Coll | 57.628 | 57.628 | 57.628 | 0.0 | 69.69 +Sort | 0.94863 | 0.94863 | 0.94863 | 0.0 | 1.15 +Comm | 0.00056522 | 0.00056522 | 0.00056522 | 0.0 | 0.00 +Modify | 0 | 0 | 0 | 0.0 | 0.00 +Output | 20.813 | 20.813 | 20.813 | 0.0 | 25.17 +MPI Sync| 0.00036906 | 0.00036906 | 0.00036906 | 0.0 | 0.00 +Other | | 5.5e-05 | | | 0.00 + +Particle moves = 200000000 (200M) +Cells touched = 212989502 (213M) +Particle comms = 0 (0K) +Boundary collides = 6493772 (6.49M) +Boundary exits = 0 (0K) +SurfColl checks = 0 (0K) +SurfColl occurs = 0 (0K) +Surf reactions = 0 (0K) +Collide attempts = 179326164 (179M) +Collide occurs = 53183155 (53.2M) +Reactions = 0 (0K) +Particles stuck = 0 +Axisymm bad moves = 0 + +Particle-moves/CPUsec/proc: 2.41849e+06 +Particle-moves/step: 1e+06 +Cell-touches/particle/step: 1.06495 +Particle comm iterations/step: 1 +Particle fraction communicated: 0 +Particle fraction colliding with boundary: 0.0324689 +Particle fraction exiting boundary: 0 +Surface-checks/particle/step: 0 +Surface-collisions/particle/step: 0 +Surf-reactions/particle/step: 0 +Collision-attempts/particle/step: 0.896631 +Collisions/particle/step: 0.265916 +Reactions/particle/step: 0 + +Particles: 1e+06 ave 1e+06 max 1e+06 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Cells: 27 ave 27 max 27 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +GhostCell: 0 ave 0 max 0 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +EmptyCell: 0 ave 0 max 0 min +Histogram: 1 0 0 0 0 0 0 0 0 0 diff --git a/examples/relax_const/log.24Aug26.mpi_4.relax_const b/examples/relax_const/log.24Aug26.mpi_4.relax_const new file mode 100644 index 000000000..629518d9d --- /dev/null +++ b/examples/relax_const/log.24Aug26.mpi_4.relax_const @@ -0,0 +1,329 @@ +SPARTA (24 Sep 2025) +Running on 4 MPI task(s) +################################################################################ +# thermal gas in a 3d box with collisions +# particles reflect off global box boundaries +# +# Note: +# - The "comm/sort” option to the “global” command is used to match MPI runs. +# - The “twopass” option is used to match Kokkos runs. +# The "comm/sort" and "twopass" options should not be used for production runs. +################################################################################ + +seed 12345 +dimension 3 +global gridcut 1.0e-5 comm/sort yes + +boundary rr rr rr + +create_box 0 0.0001 0 0.0001 0 0.0001 +Created orthogonal box = (0 0 0) to (0.0001 0.0001 0.0001) +create_grid 3 3 3 +WARNING: Could not acquire nearby ghost cells b/c grid partition is not clumped (/home/runner/work/sparta/sparta/src/grid.cpp:486) +Created 27 child grid cells + CPU time = 0.00134762 secs + create/ghost percent = 92.9014 7.09858 + +balance_grid rcb part +Balance grid migrated 24 cells + CPU time = 0.000563988 secs + reassign/sort/migrate/ghost percent = 71.4125 0.52572 11.5511 16.5106 + +species n2.species N2 +mixture air N2 vstream 0.0 0.0 0.0 temp 10000.0 trot 100.0 + +global nrho 7.07043E22 +global fnum 7.07043E5 + +collide vss air n2.vss relax constant + +create_particles air n 1000000 twopass +Created 1000000 particles + CPU time = 0.0710252 secs + +stats 1 +compute temp temp +compute T thermal/grid all all temp +compute Ttrans reduce ave c_T[1] + +compute rot grid all all trot +compute Trot reduce ave c_rot[1] + +# per-grid flux and Sonine moment diagnostics, reduced to scalars for stats + +compute ef eflux/grid all all heatx heaty heatz +compute EF reduce ave c_ef[1] c_ef[3] +compute pf pflux/grid all all momxx momyy momxy +compute PF reduce ave c_pf[1] c_pf[3] +compute sn sonine/grid all all a x 1 b xy 1 +compute SN reduce ave c_sn[1] c_sn[2] + +# ke/particle needs a deck with collisions: without them particle velocities +# never change and any reduction of it is constant for the whole run + +compute kep ke/particle +compute KE reduce max c_kep + +stats_style step cpu np nattempt ncoll c_Ttrans c_Trot c_EF[1] c_EF[2] c_PF[1] c_PF[2] c_SN[1] c_SN[2] c_KE + +timestep 1.00E-9 +run 200 +Memory usage per proc in Mbytes: + particles (ave,min,max) = 24.2188 21.875 25 + grid (ave,min,max) = 1.51379 1.51379 1.51379 + surf (ave,min,max) = 0 0 0 + modify (ave,min,max) = 0.00231743 0.00205994 0.00240326 + total (ave,min,max) = 25.7349 23.3909 26.5162 +Step CPU Np Natt Ncoll c_Ttrans c_Trot c_EF[1] c_EF[2] c_PF[1] c_PF[2] c_SN[1] c_SN[2] c_KE + 0 0 1000000 0 0 9994.2537 100.105 -233813.48 455688.6 97583.853 -91.641748 -14223274 -8.9852173e+10 2.3865477e-18 + 1 0.10673292 1000000 896599 299405 9539.5756 782.14202 -284596.45 525772.54 93184.833 -99.187786 -17283018 -4.8685893e+10 2.3865477e-18 + 2 0.21225083 1000000 896614 295432 9136.7267 1386.3505 -471822.21 228830.06 89293.886 -123.30158 -28663116 -4.0432038e+10 2.3865477e-18 + 3 0.31872551 1000000 896619 292451 8786.346 1911.9769 -579608.68 262658.44 85824.897 6.5933034 -35300432 4.0413052e+10 1.923094e-18 + 4 0.42469154 1000000 896619 289700 8478.2356 2374.1058 -613413.92 196758.67 82755.734 10.448318 -37351680 4.9983544e+10 1.9111792e-18 + 5 0.5316299 1000000 896620 287083 8205.8929 2782.6924 -510610.2 -155955.49 80088.972 -62.652221 -31138832 3.3809207e+09 1.8924278e-18 + 6 0.63864881 1000000 896619 285050 7963.3702 3146.513 -554428.89 -213770.53 77747.572 -50.062576 -33760145 -1.7734209e+09 1.7201046e-18 + 7 0.74619685 1000000 896621 282487 7752.914 3462.2153 -442586.27 -92906.702 75796.288 -45.450022 -26992989 -8.1249973e+09 1.6843123e-18 + 8 0.85321579 1000000 896625 281137 7566.2787 3742.2042 -321651.46 -221337.48 73898.307 16.838888 -19603006 -1.4744186e+10 1.6455268e-18 + 9 0.96149172 1000000 896625 278570 7399.7528 3992.0094 -213016.63 -365779.32 72321.414 -41.232932 -13014950 -3.5327265e+10 1.678781e-18 + 10 1.0684958 1000000 896619 278059 7251.029 4215.1547 -8391.5665 -215495.29 70922.89 -91.435043 -591463.23 -7.5634449e+10 1.6455268e-18 + 11 1.1766848 1000000 896620 276216 7120.0366 4411.573 80843.522 -140027.9 69529.727 -47.818481 4858346.8 -4.4245141e+10 1.6345199e-18 + 12 1.2846934 1000000 896624 275422 7005.4305 4583.4701 171510.4 -200162.43 68410.312 121.16924 10372687 4.0404876e+10 1.6345199e-18 + 13 1.3921532 1000000 896621 273469 6900.5843 4740.7915 143534.47 -51815.911 67308.18 -10.0919 8665479.7 -2.2072012e+10 1.6345199e-18 + 14 1.5013311 1000000 896626 273083 6809.1395 4877.9492 -102968.28 -159784.16 66393.518 -28.954438 -6265972.9 -7.9240972e+09 1.746565e-18 + 15 1.6114757 1000000 896625 271747 6724.827 5004.435 -17022.912 -52286.486 65609.693 -60.747388 -1044160.9 -2.9852199e+10 1.6109097e-18 + 16 1.7193347 1000000 896635 270316 6651.6499 5114.1689 -20527.748 -276316.67 64935.948 -79.877029 -1225935.1 -3.5548674e+10 1.4585156e-18 + 17 1.8283866 1000000 896626 270473 6587.8757 5209.8183 57352.687 -364524.82 64287.945 49.201045 3556886.4 3.2815493e+10 1.3839929e-18 + 18 1.9386763 1000000 896633 269755 6528.927 5298.278 10755.845 -325582.38 63721.685 30.782707 678708.41 2.2769419e+09 1.4001354e-18 + 19 2.047856 1000000 896636 268827 6480.5089 5370.9214 102254.61 -262922.3 63228.44 83.546784 6315253.1 3.8968512e+10 1.383446e-18 + 20 2.1548719 1000000 896633 268194 6435.3533 5438.6552 35938.302 -214175.43 62777.565 60.667116 2159484.6 3.8879993e+10 1.3471403e-18 + 21 2.2683021 1000000 896626 267506 6396.7171 5496.649 -18339.777 -207027.49 62436.402 91.476454 -1124021.3 2.2725105e+10 1.3471403e-18 + 22 2.3771631 1000000 896629 268407 6358.2411 5554.4083 -38406.718 -301672.09 61967.897 -13.083635 -2311026.6 -1.7829631e+10 1.5582565e-18 + 23 2.4865656 1000000 896636 267606 6324.6952 5604.7062 54917.442 -116104.6 61566.556 55.081453 3366725 9.7629299e+09 1.5582565e-18 + 24 2.5947323 1000000 896631 267232 6296.4929 5646.985 -3752.8745 -120203.24 61285.271 45.462669 -248162.64 5.3412252e+09 1.3963265e-18 + 25 2.7041009 1000000 896630 266303 6267.8019 5690.0364 -132876.96 -161074.91 61087.881 58.739453 -8120561.4 1.6150882e+10 1.4499078e-18 + 26 2.8120786 1000000 896632 266626 6242.9498 5727.28 -108967.11 78992.101 60842.62 63.39078 -6639682.9 2.4725618e+10 1.4499078e-18 + 27 2.9198885 1000000 896634 265933 6220.1563 5761.4849 -59285.691 154415.08 60654.451 63.919236 -3599351.5 8.7299342e+09 1.3406332e-18 + 28 3.0284438 1000000 896629 265601 6201.9366 5788.8046 -98435.473 247657.49 60510.786 44.276647 -5973999.1 1.4636547e+10 1.3101666e-18 + 29 3.1343385 1000000 896635 266216 6185.3794 5813.5428 85980.545 211374.72 60346.453 -34.687327 5118201 -1.7751818e+10 1.3101666e-18 + 30 3.2427114 1000000 896635 265359 6168.4652 5838.9521 98169.54 180384.94 60208.214 4.852228 5884437.9 1.6880623e+10 1.3101666e-18 + 31 3.349412 1000000 896633 265443 6155.9105 5857.7124 10162.198 274573.06 60083.756 57.635184 562249.27 3.4347505e+10 1.2109482e-18 + 32 3.4591411 1000000 896629 265663 6141.7961 5878.8762 9681.1019 164221.78 59995.914 77.994983 488230 3.1203343e+10 1.4277549e-18 + 33 3.5680527 1000000 896629 265898 6130.4563 5895.8732 -14232.008 222838.99 59834.504 5.1760914 -937225.18 -1.0257706e+10 1.1855558e-18 + 34 3.6743169 1000000 896627 264990 6122.9875 5907.1033 -152636.24 131085.23 59648.798 -85.393117 -9343974.4 -4.3866577e+10 1.2357314e-18 + 35 3.7803442 1000000 896630 265136 6111.0354 5925.0231 -238635.23 66960.536 59552.791 -72.978221 -14555148 -2.0903865e+10 1.3955077e-18 + 36 3.8887501 1000000 896631 264789 6104.59 5934.6716 -120284.82 78020.001 59511.587 13.155644 -7293501.9 2.6896086e+09 1.3955077e-18 + 37 3.9979593 1000000 896639 265712 6097.1316 5945.8552 -70538.566 117937.36 59492.368 19.396689 -4284759.9 7.8459634e+09 1.4948576e-18 + 38 4.1055351 1000000 896633 264902 6091.471 5954.3924 -42788.188 99370.564 59452.561 21.485652 -2601873.7 -5.8565116e+09 1.5321112e-18 + 39 4.2136772 1000000 896628 264305 6085.6509 5963.1392 -39517.896 -34044.135 59420.462 36.947491 -2391216.1 5.1619558e+09 1.5321112e-18 + 40 4.3218609 1000000 896627 265315 6080.4988 5970.8717 27461.138 22582.265 59309.944 60.661357 1685986.3 1.9818957e+10 1.5321112e-18 + 41 4.4285493 1000000 896623 264596 6078.0587 5974.5481 -77481.545 -29651.979 59282.549 1.8571296 -4692510 -7.6902989e+09 1.2037528e-18 + 42 4.5371946 1000000 896619 264842 6073.8826 5980.8292 -46333.148 -22337.056 59263.831 -105.75836 -2811649.2 -4.6027547e+10 1.3203669e-18 + 43 4.6443178 1000000 896618 264328 6068.2345 5989.2724 -24745.999 -33094.164 59208.265 -154.18771 -1486120.1 -5.5829445e+10 1.2677092e-18 + 44 4.7514466 1000000 896617 264784 6061.4252 5999.5045 -90955.293 -23891.015 59166.382 -78.543208 -5510593.8 -3.8518454e+10 1.2677092e-18 + 45 4.8589114 1000000 896630 264289 6059.6322 6002.1986 -91426.405 -61137.788 59166.606 -52.598308 -5551836.4 -4.7390975e+10 1.3980942e-18 + 46 4.9688177 1000000 896621 264371 6057.8717 6004.8957 -50654.334 66959.016 59129.367 -105.12872 -3102853.7 -4.873587e+10 1.3980942e-18 + 47 5.0768295 1000000 896622 264386 6056.2604 6007.267 159185.07 -7472.135 59144.598 -47.819803 9689114.4 -2.3120433e+10 1.2408015e-18 + 48 5.1854692 1000000 896618 264857 6052.4143 6013.0622 71121.336 72350.724 59085.432 1.3371776 4322989.5 -1.4063821e+10 1.2408015e-18 + 49 5.2927503 1000000 896624 264504 6052.1244 6013.5134 -2498.6551 13898.894 59146.219 0.72564982 -155785.63 1.1054647e+10 1.2144445e-18 + 50 5.400007 1000000 896634 264237 6049.3917 6017.5837 35204.76 45125.448 59027.728 -38.584577 2120679 -9.4710804e+09 1.3601216e-18 + 51 5.5075869 1000000 896629 264295 6046.8249 6021.4743 128231.07 74830.457 58967.235 -19.902762 7769605.2 1.3436632e+10 1.3601216e-18 + 52 5.6168022 1000000 896630 264183 6049.2725 6017.841 145642.81 10047.53 58941.468 25.091678 8830620.9 1.3932335e+10 1.1770016e-18 + 53 5.7236992 1000000 896630 264598 6046.6698 6021.6705 153549.14 5411.4014 59026.966 22.247874 9318248.9 1.0328201e+10 1.3508986e-18 + 54 5.8303779 1000000 896622 263770 6045.0473 6024.0747 84910.031 -53237.461 58982.179 -51.580584 5177078.1 -1.349351e+10 1.3508986e-18 + 55 5.9389091 1000000 896630 264333 6044.5284 6024.9437 127332.55 92256.695 59009.601 5.5648376 7807346.3 21940802 1.3508986e-18 + 56 6.0450539 1000000 896630 264354 6046.521 6021.9993 -24373.859 -70974.746 59035.262 -4.0422434 -1452936.4 7.1611242e+09 1.1482245e-18 + 57 6.1540044 1000000 896628 264892 6045.7361 6023.1639 10228.158 -37180.888 59004.752 1.5413074 643938.34 1.0340826e+10 1.3689822e-18 + 58 6.2622729 1000000 896636 263557 6046.9575 6021.2898 22988.491 -63927.138 59157.126 -47.603211 1434012.1 -1.0690287e+10 1.2239317e-18 + 59 6.3707617 1000000 896625 263829 6043.6509 6026.2014 88736.074 -28171.875 59043.335 -74.99522 5422956.3 -2.8570166e+10 1.3033853e-18 + 60 6.4791001 1000000 896628 264781 6041.4206 6029.5182 50332.887 61518.426 58920.804 -123.29305 3086497.7 -3.4799274e+10 1.2381345e-18 + 61 6.586795 1000000 896631 263964 6042.2779 6028.2177 163308.18 40138.786 59000.655 -81.980634 9959915.3 -3.6851943e+10 1.2531139e-18 + 62 6.6934957 1000000 896628 264455 6040.3274 6031.2021 65935.665 -59601.054 58986.062 -52.799515 4011074.9 -1.2928798e+10 1.2588285e-18 + 63 6.8007226 1000000 896628 264327 6040.1995 6031.3409 152834.17 54420.437 59031.19 -6.7594096 9272288 9.1999623e+09 1.2588285e-18 + 64 6.9083505 1000000 896630 263548 6039.0765 6033.0045 305858.61 160592.36 58946.065 -11.497613 18609192 -5.4460703e+09 1.133289e-18 + 65 7.0161994 1000000 896627 264376 6037.7688 6035.0333 206426.04 155241.25 58914.006 -8.8522708 12553221 6.5286816e+09 1.2262602e-18 + 66 7.1229587 1000000 896624 264692 6038.6108 6033.7604 104301.45 239240.64 58971.236 9.5370916 6360329.2 1.123421e+10 1.2262602e-18 + 67 7.2305992 1000000 896630 264095 6036.379 6037.1025 50111.093 119599.37 58993.241 61.790971 3052641.3 2.2037838e+10 1.2488925e-18 + 68 7.3377532 1000000 896627 263876 6033.5442 6041.4087 20486.83 136726.83 58923.451 22.469023 1220417 -4.1221561e+09 1.2283013e-18 + 69 7.4476859 1000000 896636 264586 6033.952 6040.813 13242.75 127687.76 58854.26 13.949745 820296.37 -3.0134942e+10 1.2283013e-18 + 70 7.5632482 1000000 896627 263942 6033.2418 6041.8937 69584.612 152575.49 58839.039 33.366185 4282527 -1.9600498e+10 1.3279415e-18 + 71 7.672438 1000000 896631 263598 6033.8099 6041.0128 104142.88 76940.411 58820.604 -49.098493 6318950.4 -4.3769819e+10 1.3279415e-18 + 72 7.7804417 1000000 896619 263851 6033.7713 6041.0492 230341.48 -49271.458 58964.857 -46.358042 14030541 -4.2482891e+10 1.3186119e-18 + 73 7.8883246 1000000 896628 264424 6032.3147 6043.2527 225019.08 -152587.07 58957.267 -103.79738 13676645 -6.4750613e+10 1.3186119e-18 + 74 7.9973902 1000000 896621 263918 6030.9976 6045.2712 252711.6 -103235.7 58897.756 -80.450289 15354377 -5.6954768e+10 1.4632274e-18 + 75 8.1043316 1000000 896627 263966 6034.2243 6040.3865 105525.2 -127953.25 58878.564 -35.309296 6379402.4 -2.6922956e+10 1.2864107e-18 + 76 8.2170478 1000000 896621 264197 6034.5422 6039.8904 30633.344 -235465.34 58938.201 -95.138568 1833889.4 -5.0601733e+10 1.2866251e-18 + 77 8.3238987 1000000 896625 264698 6033.6117 6041.3144 -135281.76 -137393.37 58921.055 -80.160601 -8270137 -3.9535663e+10 1.227809e-18 + 78 8.4328746 1000000 896618 263552 6032.9258 6042.3802 -149523.29 -23248.642 58923.666 -21.989134 -9091512.7 -2.715489e+09 1.227809e-18 + 79 8.5396408 1000000 896622 264013 6035.5855 6038.3346 -395953.9 98278.077 59014.505 -52.285582 -24093216 -1.2445797e+10 1.227809e-18 + 80 8.64758 1000000 896618 263837 6032.8641 6042.3968 -365589.96 107889.31 59015.722 -13.177626 -22256089 -1.3934221e+10 1.3079848e-18 + 81 8.7551033 1000000 896618 264838 6030.3618 6046.1336 -204772.47 56433.984 58911.539 18.760703 -12493118 -2.4117736e+09 1.355739e-18 + 82 8.8630311 1000000 896620 263651 6031.9665 6043.6829 -114924.64 116146.77 58938.501 -57.390376 -7060308.7 -1.7844021e+10 1.250707e-18 + 83 8.9721004 1000000 896623 263471 6029.8146 6046.8882 77212.416 145411.37 58884.95 -62.848746 4683169.4 2.5543931e+09 1.250707e-18 + 84 9.0795976 1000000 896622 263624 6030.4266 6045.989 -3566.5594 -113858.9 58929.581 -160.08958 -225610.34 -5.6924722e+10 1.2795999e-18 + 85 9.1884597 1000000 896619 264156 6029.2946 6047.6979 -74158.657 -221611.8 58863.802 -125.72952 -4515221.9 -3.7615173e+10 1.2795999e-18 + 86 9.2973414 1000000 896620 264585 6027.8506 6049.8827 -37785.848 -94280.282 58841.868 -54.748792 -2262388.2 -6.8965989e+09 1.3882204e-18 + 87 9.40506 1000000 896624 264097 6029.8796 6046.8822 -187165.41 -18298.481 58903.351 2.9784569 -11373878 4.1834277e+09 1.4337942e-18 + 88 9.512409 1000000 896623 264010 6032.4053 6043.0708 55852.485 -60526.049 58905.974 -76.997672 3398117.3 -2.3044863e+10 1.4337942e-18 + 89 9.6260958 1000000 896629 265232 6035.4914 6038.4503 2938.2909 -1260.3672 58889.807 -66.951023 209360.12 -2.0101461e+10 1.2281533e-18 + 90 9.7334578 1000000 896629 264623 6036.3331 6037.1893 -39332.314 -19176.248 58944.411 -78.17318 -2346271.3 -3.3747725e+10 1.3534682e-18 + 91 9.840544 1000000 896627 264477 6034.5214 6039.9993 -83227.739 -37820.232 59009.503 -48.586375 -5038506.6 -9.7612075e+09 1.2027141e-18 + 92 9.9487239 1000000 896628 263937 6036.0845 6037.6144 -150975.14 -101819.85 59043.55 -9.930439 -9131275.3 5.0336635e+09 1.444083e-18 + 93 10.057043 1000000 896628 264957 6036.5161 6036.9432 13676.607 -78143.558 59048.184 -48.805763 819573.44 -1.0648151e+10 1.3103473e-18 + 94 10.16582 1000000 896630 264116 6039.1072 6033.0572 36586.392 -175992.33 58989.24 -101.22548 2234520.1 -4.204521e+10 1.1922698e-18 + 95 10.27365 1000000 896635 264202 6037.6931 6035.1812 44065.198 -48267.756 59010.32 -81.840994 2665817.1 -3.18504e+10 1.4057752e-18 + 96 10.382511 1000000 896628 264010 6035.9255 6037.8038 -195509.08 -108403.96 58893.189 -16.796667 -11938474 -3.2762278e+09 1.2003581e-18 + 97 10.490017 1000000 896636 263755 6034.5645 6039.8552 -20096.007 -180036.13 58880.884 -53.008042 -1289599.6 -1.3156577e+10 1.3372087e-18 + 98 10.598181 1000000 896628 264260 6034.8816 6039.3983 117329.04 -175162.52 58914.414 -63.616016 7161818.2 -2.3018144e+10 1.2778931e-18 + 99 10.707977 1000000 896625 263969 6038.4592 6034.0416 175717.41 -24257.105 59017.309 -62.235194 10711378 -1.0184828e+10 1.242265e-18 + 100 10.81628 1000000 896628 264714 6037.7418 6035.1253 210363.17 66962.432 58941.376 -59.513728 12804669 -1.0415214e+10 1.242265e-18 + 101 10.92524 1000000 896631 263761 6036.4215 6037.06 109358.59 103389.48 58880.523 -3.7709231 6617096.5 1.0184094e+10 1.3102058e-18 + 102 11.033794 1000000 896627 264133 6034.7014 6039.6521 83140.614 15336.064 58865.139 -40.804945 5040643 -6.7412215e+09 1.2506039e-18 + 103 11.139264 1000000 896633 264463 6036.8016 6036.4987 156605.22 106826.13 58874.035 15.186886 9562184 1.4057136e+10 1.2885365e-18 + 104 11.246007 1000000 896632 264561 6037.4291 6035.548 191321.64 -20096.249 58748.929 -39.739894 11626362 -9.7225526e+09 1.2629834e-18 + 105 11.35261 1000000 896634 264760 6036.869 6036.3424 183329.58 44972.88 58871.058 -21.295065 11130188 -1.3453094e+10 1.3818685e-18 + 106 11.461089 1000000 896636 263880 6037.0699 6035.9909 115214.4 24393.843 58882.216 -77.217201 6978538.3 -1.9611685e+10 1.1646689e-18 + 107 11.571239 1000000 896633 263948 6036.8694 6036.3785 283009.54 -6237.6512 58903.37 -18.376024 17200366 -4.3231054e+08 1.2998117e-18 + 108 11.679114 1000000 896632 264698 6035.4474 6038.6518 259686.02 -32417.146 58878.799 -69.907325 15756802 -3.9319005e+10 1.2305156e-18 + 109 11.787864 1000000 896628 264320 6036.8621 6036.4728 148384.06 26808.435 58923.778 -50.19686 9028230.3 -1.5276574e+10 1.207874e-18 + 110 11.898265 1000000 896632 264317 6036.6538 6036.861 94412.473 -67192.532 58972.466 -75.161929 5760126.6 -3.2323245e+10 1.2073039e-18 + 111 12.008834 1000000 896631 264277 6036.1744 6037.5779 102117.76 -122649.19 58873.935 -54.145779 6175344.3 -1.3499478e+10 1.3506167e-18 + 112 12.116812 1000000 896626 264410 6037.3091 6035.8133 215797.27 23892.516 58859.677 -26.508149 13123041 4.2159858e+09 1.2692859e-18 + 113 12.225349 1000000 896631 265137 6034.243 6040.4335 244214.77 -143792.13 58819.208 29.177168 14910042 3.6008641e+10 1.123491e-18 + 114 12.334513 1000000 896633 263750 6032.5273 6042.9679 182776.28 -170473.65 58924.786 6.2848643 11091360 1.8267172e+10 1.2420011e-18 + 115 12.443488 1000000 896641 264437 6034.4421 6040.0583 224941.66 -66497.21 58997.525 94.712894 13685888 5.3353284e+10 1.2935552e-18 + 116 12.552336 1000000 896633 263262 6031.4504 6044.5121 133801.57 -55016.627 58975.479 134.5007 8145786 6.1510961e+10 1.2930777e-18 + 117 12.661152 1000000 896635 263829 6031.5138 6044.448 125422.05 -92989.575 58915.821 129.65825 7596094.7 5.4257131e+10 1.4493237e-18 + 118 12.770503 1000000 896635 263983 6030.4796 6045.9723 140235.22 -117051.69 58906.584 76.757421 8528802.1 3.2572797e+10 1.4493237e-18 + 119 12.879206 1000000 896629 264386 6030.554 6045.8773 50208.519 -66994.84 58867.661 138.02713 3066447.6 4.8441464e+10 1.4259349e-18 + 120 12.988204 1000000 896631 263746 6030.474 6045.9833 74027.393 12068.179 58880.333 60.843895 4544027.3 2.4240935e+10 1.2378259e-18 + 121 13.097382 1000000 896632 264164 6031.4771 6044.4587 161321.21 -62480.031 58928.037 64.07702 9854033.8 2.5418047e+10 1.4123914e-18 + 122 13.204065 1000000 896628 263085 6029.2667 6047.7899 127675.63 -173130.19 58945.278 52.814155 7828484.4 7.7456078e+09 1.4123914e-18 + 123 13.31189 1000000 896628 264433 6029.6155 6047.2981 132189.6 -159046.25 58905.718 2.3093319 8080644.7 -7.3096051e+09 1.4929957e-18 + 124 13.418687 1000000 896632 264729 6030.2319 6046.3404 242452.31 -112682.31 58963.188 66.445633 14768749 1.4498355e+10 1.222791e-18 + 125 13.526559 1000000 896634 264663 6032.3092 6043.2369 192562.19 -197964.94 59073.955 8.8816402 11750093 2.2607047e+09 1.2143376e-18 + 126 13.635577 1000000 896632 263751 6033.3689 6041.6983 143411.18 -129028.24 59092.761 -22.392734 8753253.5 -8.7249471e+09 1.2143376e-18 + 127 13.744088 1000000 896631 264096 6032.1389 6043.5094 152424.46 -139813.51 59067.558 -9.1265765 9302783.3 -3.4129215e+09 1.2554736e-18 + 128 13.85113 1000000 896638 264017 6032.2762 6043.2334 133184.01 88089.138 59001.831 9.4650404 8160387.6 4.8381985e+09 1.2953741e-18 + 129 13.957346 1000000 896635 263385 6032.3967 6043.0396 114659.5 -112462.6 58919.587 -43.763363 7002335 -2.9048282e+10 1.3795252e-18 + 130 14.064417 1000000 896631 263716 6031.1905 6044.7909 47166.037 -154443.8 58935.902 5.5196086 2914994.2 9.1990337e+09 1.4606257e-18 + 131 14.170957 1000000 896628 264767 6030.0789 6046.5001 -31031.396 -236351.08 58889.705 57.549362 -1855647.6 2.4253781e+10 1.1836554e-18 + 132 14.281566 1000000 896626 264542 6031.012 6045.1426 -93026.712 -153229.95 58886.893 54.194266 -5660595.9 1.4251047e+10 1.2900145e-18 + 133 14.388106 1000000 896623 264379 6030.8312 6045.3779 -375158.54 -120101.7 58866.954 75.462046 -22818726 2.5991939e+10 1.3118826e-18 + 134 14.494631 1000000 896629 263475 6032.0232 6043.62 -262068.26 -100415.08 58911.172 7.2639332 -15934550 9.1678055e+08 1.2413489e-18 + 135 14.602808 1000000 896630 263361 6029.9771 6046.73 -290239.45 -168420.06 58850.779 -11.074207 -17653449 -4.3469953e+09 1.1991724e-18 + 136 14.712079 1000000 896626 264965 6029.5251 6047.4024 -81029.087 -209306.88 58908.751 2.3193673 -4960031.9 6.4360567e+09 1.2758063e-18 + 137 14.821037 1000000 896627 263730 6029.5342 6047.4122 10069.933 -179877.01 58838.368 81.553202 629399.77 2.755828e+10 1.2039354e-18 + 138 14.929902 1000000 896628 264275 6031.2314 6044.892 18876.846 -195152.4 58845.78 52.343403 1177476.9 2.0974018e+10 1.397026e-18 + 139 15.03831 1000000 896626 264340 6029.8569 6046.9749 -64028.942 -101404.76 58738.175 22.740385 -3868156.7 1.1909006e+10 1.2039354e-18 + 140 15.144834 1000000 896628 264367 6031.4428 6044.5997 -22377.501 -47632.644 58840.368 68.765608 -1383158.6 1.8337379e+10 1.195118e-18 + 141 15.25429 1000000 896629 264624 6030.3967 6046.2037 -30959.125 -119537.06 58826.435 82.48379 -1878306.3 1.5612018e+10 1.2401761e-18 + 142 15.361013 1000000 896636 264516 6030.8883 6045.4098 -81804.343 -145681.85 58855.693 51.25407 -4906562.6 1.8104236e+10 1.2401761e-18 + 143 15.468725 1000000 896634 264034 6031.3879 6044.6314 -199894.06 -125100.88 58905.613 -17.617278 -12143484 -9.1912157e+08 1.4312841e-18 + 144 15.578416 1000000 896637 264542 6028.5124 6048.9139 -242788.16 -128601.5 58914.423 -0.61445766 -14712941 -1.3938342e+09 1.2644736e-18 + 145 15.688264 1000000 896634 264821 6030.991 6045.1895 -278686.64 -30483.551 58907.309 -17.456786 -16909673 -2.3247986e+09 1.2247381e-18 + 146 15.797777 1000000 896638 264097 6031.3226 6044.727 -208902.9 58544.855 58892.482 35.206716 -12728014 1.3845046e+10 1.2247381e-18 + 147 15.909435 1000000 896636 264473 6031.6841 6044.1748 -145113.48 -24596.474 58888.021 -28.553684 -8838867.8 -2.1622709e+10 1.2271368e-18 + 148 16.017934 1000000 896634 263423 6030.7642 6045.589 -73770.636 -66951.401 58824.588 -48.612275 -4515214.9 -8.8819008e+09 1.2707337e-18 + 149 16.124058 1000000 896637 264454 6031.8829 6043.925 -48328.452 -46545.546 58949.538 -64.086325 -2956267.1 -2.8662989e+10 1.4904112e-18 + 150 16.232215 1000000 896634 264143 6032.7179 6042.6524 51446.269 -179703.6 58961.887 -111.32369 3143217.7 -4.5244061e+10 1.3724813e-18 + 151 16.341554 1000000 896634 264373 6032.5153 6042.9515 68002.869 -300220.2 58846.435 -62.626575 4158785.2 -2.5871301e+10 1.253007e-18 + 152 16.449392 1000000 896632 264048 6030.8128 6045.5394 103909.36 -181269.23 58846.751 -6.5838164 6333789.4 7.4998227e+09 1.253007e-18 + 153 16.558236 1000000 896634 263240 6030.863 6045.4497 138772.21 -53385.731 58846.841 -42.130681 8472190.7 -1.6171056e+10 1.2547991e-18 + 154 16.665723 1000000 896631 264486 6035.1283 6039.0931 33967.55 -1071.3274 58878.134 -22.927189 2077966.6 -1.3072925e+10 1.2593113e-18 + 155 16.772218 1000000 896630 264034 6034.5445 6039.9186 18963.709 -56687.421 58894.617 -31.57218 1160228.1 -3.3645751e+10 1.3956827e-18 + 156 16.881121 1000000 896629 263892 6032.1884 6043.4451 -34565.885 -248675.77 58790.173 -11.019185 -2083337.1 -3.0046417e+10 1.3956827e-18 + 157 16.988764 1000000 896630 264573 6033.9467 6040.7578 30491.443 -86532.107 58860.394 4.5363528 1814273 -2.8882943e+10 1.3703543e-18 + 158 17.107601 1000000 896634 264240 6035.3631 6038.6553 5834.9599 -79624.219 58900.527 -33.099077 353873.44 -2.1108353e+10 1.5209123e-18 + 159 17.223284 1000000 896637 264029 6036.2641 6037.2751 -93641.557 14934.466 59000.956 -48.776681 -5688057.7 -2.8120952e+10 1.5209123e-18 + 160 17.33144 1000000 896635 264698 6034.5455 6039.8702 -150300.05 -69715.61 58986.545 -8.9986232 -9084572.6 -4.1564089e+09 1.5338686e-18 + 161 17.439467 1000000 896635 263453 6033.2087 6041.9088 -156442.2 -37266.506 58933.718 -56.642776 -9494825.6 -3.0878266e+10 1.2534925e-18 + 162 17.548651 1000000 896634 265151 6033.8536 6040.9448 -71283.752 -8139.4783 58889.256 -7.7154245 -4315847.9 6.5481784e+09 1.2735288e-18 + 163 17.656007 1000000 896624 264687 6035.6613 6038.224 -105973.72 -94766.771 58967.92 -17.476543 -6396643 -8.8878814e+09 1.2735288e-18 + 164 17.765142 1000000 896629 263645 6035.5642 6038.3766 -133726.15 -86066.439 58960.948 53.703434 -8099905.5 2.3481556e+10 1.2483211e-18 + 165 17.874568 1000000 896624 263656 6034.8644 6039.35 -170739.93 -68324.314 58931.321 23.444675 -10336079 1.1756152e+10 1.18937e-18 + 166 17.983565 1000000 896620 263846 6036.182 6037.3842 -296502.99 -10495.249 58921.644 -64.488762 -18043682 -2.3910366e+10 1.2149327e-18 + 167 18.089609 1000000 896623 264282 6034.6052 6039.735 -149074.92 64722.646 58882.538 2.1598411 -9072068.7 1.0555455e+09 1.2082812e-18 + 168 18.197751 1000000 896620 264260 6033.6253 6041.1071 -158861.1 35508.821 58916.932 17.794987 -9654940.3 1.5282238e+10 1.1679328e-18 + 169 18.305734 1000000 896619 264821 6037.0649 6036.0126 -45142.863 111157.86 58897.807 11.072499 -2724668.4 7.4352251e+09 1.1679328e-18 + 170 18.413113 1000000 896623 264405 6038.9305 6033.262 33800.662 226719.41 58923.715 4.2240964 2052348.2 -2.0413855e+09 1.35698e-18 + 171 18.524186 1000000 896621 263558 6039.3288 6032.6241 -46153.147 179207.49 58953.644 15.417723 -2777383.5 7.3182002e+09 1.26143e-18 + 172 18.631765 1000000 896627 264508 6039.4237 6032.458 -125595.93 105334.89 58972.97 31.103013 -7599991.3 1.868915e+10 1.4452297e-18 + 173 18.738446 1000000 896632 264292 6040.0065 6031.5917 -255204.62 75690.864 59016.947 10.459721 -15497919 7.3319283e+09 1.3835813e-18 + 174 18.845358 1000000 896621 264012 6038.7924 6033.4351 -255051.51 137946.2 58970.978 -21.950335 -15495360 -1.4106464e+10 1.3239542e-18 + 175 18.95348 1000000 896630 264351 6038.8023 6033.3722 -125004.54 52341.707 58940.674 -29.006041 -7562713 -3.1991534e+10 1.3582224e-18 + 176 19.059723 1000000 896633 263997 6039.8179 6031.8509 -273773.91 158994.6 59021.111 -61.288263 -16657393 -3.8100257e+10 1.3582224e-18 + 177 19.16614 1000000 896628 264633 6041.5701 6029.222 -379651.41 290796.37 59052.843 -89.805578 -23064116 -5.4685869e+10 1.3347668e-18 + 178 19.275282 1000000 896628 265073 6041.1344 6029.8675 -200904.36 237470.77 58989.815 -44.214899 -12170753 -2.5738349e+10 1.3347668e-18 + 179 19.388442 1000000 896620 264776 6042.8131 6027.3674 -345332.96 106870.44 58969.058 -13.93561 -21042202 -2.9372308e+10 1.3090223e-18 + 180 19.499199 1000000 896624 263915 6043.9904 6025.546 -243552.44 58437.549 58970.951 -15.857183 -14845880 -4.1485192e+10 1.3887518e-18 + 181 19.608051 1000000 896629 263884 6042.0295 6028.5962 -235765.03 -65959.72 59048.168 -12.515947 -14358286 -1.729807e+10 1.479709e-18 + 182 19.716546 1000000 896628 264520 6039.2423 6032.7973 -54260.857 16403.261 59029.004 58.45121 -3351369.7 2.0778151e+10 1.479709e-18 + 183 19.826356 1000000 896627 264474 6039.2182 6032.8655 102029.63 -57779.258 58886.515 39.746475 6151132.3 2.1392084e+10 1.2252435e-18 + 184 19.937674 1000000 896626 264141 6040.2635 6031.2983 69520.561 74498.302 58933.884 -41.03266 4130693.3 -8.8658259e+09 1.3237084e-18 + 185 20.048218 1000000 896621 263714 6039.9741 6031.74 -31175.47 91679.033 58927.934 -60.828499 -1968689.7 -1.9833877e+10 1.3620406e-18 + 186 20.156629 1000000 896633 263858 6037.8852 6034.8622 27938.408 47697.856 58945.235 -87.039444 1691675.2 -2.9837739e+10 1.3767911e-18 + 187 20.273096 1000000 896630 263784 6038.7779 6033.5476 17732.174 -79360.019 58975.745 -65.471223 1030353 -2.9550366e+10 1.3767911e-18 + 188 20.382495 1000000 896634 263609 6037.2085 6035.9173 156129.6 -234319.46 59061.353 -110.75645 9446226.9 -6.3648887e+10 1.296546e-18 + 189 20.491159 1000000 896633 264517 6036.0028 6037.7141 -89392.758 -325535.49 58959.597 -82.628646 -5485926.1 -2.8255933e+10 1.2265501e-18 + 190 20.603177 1000000 896639 264011 6034.3488 6040.2049 52195.668 -250255.87 59094.442 -80.77182 3100043.7 -2.3242763e+10 1.1926694e-18 + 191 20.712246 1000000 896635 264273 6035.6436 6038.2899 24154.949 -176220.44 59060.6 5.7606437 1477381.4 7.627007e+09 1.2277228e-18 + 192 20.820026 1000000 896628 264914 6035.6633 6038.2615 73633.755 -46952.808 59020.138 36.654904 4494262.4 3.0260153e+10 1.2406218e-18 + 193 20.929504 1000000 896636 265223 6035.7902 6038.0641 4438.89 -83126.842 58901.751 -46.130636 292142.35 -6.3862604e+09 1.3924366e-18 + 194 21.038493 1000000 896644 264378 6036.7601 6036.5303 -114166.73 78801.853 58870.539 -18.910724 -6916103.5 -7.2878801e+09 1.6325559e-18 + 195 21.147874 1000000 896631 265049 6036.4729 6036.9612 -75688.079 141396.63 58861.487 -67.962319 -4594710.7 -1.4023929e+10 1.2277228e-18 + 196 21.255575 1000000 896632 264350 6038.6801 6033.6968 -95934.51 133070.23 58921.082 -40.712553 -5865875.2 -1.0999675e+10 1.2653771e-18 + 197 21.365261 1000000 896629 264067 6036.2151 6037.377 -66199.007 214077.3 58984.792 92.189581 -4008723.7 4.8047027e+10 1.2629421e-18 + 198 21.473689 1000000 896630 263427 6036.908 6036.3067 -45774.272 89149.394 59050.453 57.111968 -2808390.8 3.7344146e+10 1.2601795e-18 + 199 21.582077 1000000 896635 263588 6035.1967 6038.8577 79880.527 155766.31 58939.01 51.208247 4787282.9 2.1087974e+10 1.2601795e-18 + 200 21.690507 1000000 896627 264698 6034.7771 6039.4656 -43431.94 149965.2 58978.354 92.898201 -2695099.9 3.6280455e+10 1.2601795e-18 +Loop time of 21.6907 on 4 procs for 200 steps with 1000000 particles +Performance: 9.221 timesteps/s, 9.221 Mparticle-step/s + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Move | 0.79781 | 0.90206 | 0.93938 | 6.3 | 4.16 +Coll | 11.197 | 12.757 | 13.412 | 25.6 | 58.82 +Sort | 0.59566 | 0.63345 | 0.65285 | 2.8 | 2.92 +Comm | 0.28507 | 0.29132 | 0.29739 | 1.1 | 1.34 +Modify | 0 | 0 | 0 | 0.0 | 0.00 +Output | 6.3763 | 7.0502 | 8.6377 | 35.0 | 32.50 +MPI Sync| 0.010839 | 0.056184 | 0.16575 | 26.8 | 0.26 +Other | | 7.283e-05 | | | 0.00 + +Particle moves = 200000000 (200M) +Cells touched = 212988680 (213M) +Particle comms = 7502635 (7.5M) +Boundary collides = 6496235 (6.5M) +Boundary exits = 0 (0K) +SurfColl checks = 0 (0K) +SurfColl occurs = 0 (0K) +Surf reactions = 0 (0K) +Collide attempts = 179325686 (179M) +Collide occurs = 53181041 (53.2M) +Reactions = 0 (0K) +Particles stuck = 0 +Axisymm bad moves = 0 + +Particle-moves/CPUsec/proc: 2.30514e+06 +Particle-moves/step: 1e+06 +Cell-touches/particle/step: 1.06494 +Particle comm iterations/step: 1 +Particle fraction communicated: 0.0375132 +Particle fraction colliding with boundary: 0.0324812 +Particle fraction exiting boundary: 0 +Surface-checks/particle/step: 0 +Surface-collisions/particle/step: 0 +Surf-reactions/particle/step: 0 +Collision-attempts/particle/step: 0.896628 +Collisions/particle/step: 0.265905 +Reactions/particle/step: 0 + +Particles: 250000 ave 259675 max 222168 min +Histogram: 1 0 0 0 0 0 0 0 0 3 +Cells: 6.75 ave 7 max 6 min +Histogram: 1 0 0 0 0 0 0 0 0 3 +GhostCell: 20.25 ave 21 max 20 min +Histogram: 3 0 0 0 0 0 0 0 0 1 +EmptyCell: 0 ave 0 max 0 min +Histogram: 4 0 0 0 0 0 0 0 0 0 diff --git a/examples/relax_variable/log.24Aug26.mpi_1.relax_variable b/examples/relax_variable/log.24Aug26.mpi_1.relax_variable new file mode 100644 index 000000000..a4b3fd53b --- /dev/null +++ b/examples/relax_variable/log.24Aug26.mpi_1.relax_variable @@ -0,0 +1,313 @@ +SPARTA (24 Sep 2025) +Running on 1 MPI task(s) +################################################################################ +# thermal gas in a 3d box with collisions +# particles reflect off global box boundaries +# +# Note: +# - The "comm/sort” option to the “global” command is used to match MPI runs. +# - The “twopass” option is used to match Kokkos runs. +# The "comm/sort" and "twopass" options should not be used for production runs. +################################################################################ + +seed 12345 +dimension 3 +global gridcut 1.0e-5 comm/sort yes + +boundary rr rr rr + +create_box 0 0.0001 0 0.0001 0 0.0001 +Created orthogonal box = (0 0 0) to (0.0001 0.0001 0.0001) +create_grid 3 3 3 +Created 27 child grid cells + CPU time = 0.0012249 secs + create/ghost percent = 97.1335 2.86652 + +balance_grid rcb part +Balance grid migrated 0 cells + CPU time = 0.000104776 secs + reassign/sort/migrate/ghost percent = 73.9625 0.115484 19.2496 6.67233 + +species n2.species N2 +mixture air N2 vstream 0.0 0.0 0.0 temp 10000.0 trot 100.0 + +global nrho 7.07043E22 +global fnum 7.07043E5 + +collide vss air n2.vss relax variable + +create_particles air n 1000000 twopass +Created 1000000 particles + CPU time = 0.264496 secs + +stats 1 +compute temp temp +compute T thermal/grid all all temp +compute Ttrans reduce ave c_T[1] + +compute rot grid all all trot +compute Trot reduce ave c_rot[1] + +stats_style step cpu np nattempt ncoll c_Ttrans c_Trot + +timestep 1.00E-9 +run 200 +Memory usage per proc in Mbytes: + particles (ave,min,max) = 96.875 96.875 96.875 + grid (ave,min,max) = 1.51379 1.51379 1.51379 + surf (ave,min,max) = 0 0 0 + modify (ave,min,max) = 0.00205994 0.00205994 0.00205994 + total (ave,min,max) = 98.3909 98.3909 98.3909 +Step CPU Np Natt Ncoll c_Ttrans c_Trot + 0 0 1000000 0 0 9993.6754 99.967421 + 1 0.3406253 1000000 896601 300973 9784.1502 414.2944 + 2 0.69071387 1000000 896616 298104 9587.3338 709.41629 + 3 1.0434935 1000000 896611 296983 9399.188 991.66194 + 4 1.3997666 1000000 896617 296377 9224.5143 1253.7345 + 5 1.7571053 1000000 896620 293898 9057.5599 1504.1431 + 6 2.1083897 1000000 896618 293023 8900.4736 1739.7851 + 7 2.457563 1000000 896618 290907 8748.9865 1967.0621 + 8 2.8108727 1000000 896619 289123 8608.6388 2177.4884 + 9 3.1654028 1000000 896624 289113 8476.6874 2375.3613 + 10 3.5257214 1000000 896620 287689 8351.2736 2563.5305 + 11 3.8778032 1000000 896623 287077 8230.4991 2744.766 + 12 4.2371418 1000000 896622 285009 8116.8469 2915.2369 + 13 4.5845133 1000000 896620 284770 8010.0397 3075.53 + 14 4.9419506 1000000 896624 283717 7907.1216 3229.9693 + 15 5.3242477 1000000 896623 282555 7811.5112 3373.3264 + 16 5.688025 1000000 896632 282305 7721.6811 3507.9968 + 17 6.0543393 1000000 896636 281578 7635.7027 3636.9007 + 18 6.4279781 1000000 896632 279958 7554.6846 3758.4197 + 19 6.7945776 1000000 896633 280148 7473.8545 3879.6519 + 20 7.160291 1000000 896632 278992 7399.9664 3990.4236 + 21 7.525023 1000000 896632 277992 7329.3213 4096.3869 + 22 7.8906324 1000000 896630 277631 7262.7697 4196.246 + 23 8.2568641 1000000 896627 276522 7199.8018 4290.7288 + 24 8.620448 1000000 896630 275621 7140.4036 4379.8652 + 25 8.9780259 1000000 896633 275317 7085.9778 4461.5328 + 26 9.3326314 1000000 896630 275087 7032.2172 4542.1958 + 27 9.6705064 1000000 896634 274077 6982.1019 4617.3342 + 28 9.9724737 1000000 896629 273142 6935.5066 4687.2453 + 29 10.314925 1000000 896632 274434 6891.4482 4753.3342 + 30 10.65296 1000000 896629 273640 6848.1559 4818.2612 + 31 10.990306 1000000 896631 272044 6806.4885 4880.7061 + 32 11.370268 1000000 896631 272461 6768.4356 4937.7653 + 33 11.724874 1000000 896630 272176 6731.7348 4992.8192 + 34 12.069131 1000000 896627 271126 6695.852 5046.7167 + 35 12.4086 1000000 896629 270903 6663.5992 5095.0143 + 36 12.726751 1000000 896637 270440 6632.0035 5142.3647 + 37 13.057085 1000000 896632 270739 6602.1728 5187.105 + 38 13.374593 1000000 896636 270692 6574.9621 5227.9028 + 39 13.701834 1000000 896625 269774 6549.3657 5266.3252 + 40 14.072039 1000000 896635 269713 6523.7206 5304.834 + 41 14.406591 1000000 896627 270070 6500.956 5339.0192 + 42 14.735141 1000000 896626 269801 6477.9648 5373.5166 + 43 15.063543 1000000 896629 268931 6456.7753 5405.3272 + 44 15.351611 1000000 896624 267948 6436.1949 5436.2258 + 45 15.696123 1000000 896629 269727 6417.7069 5463.9514 + 46 16.043966 1000000 896627 268633 6395.9307 5496.6028 + 47 16.410703 1000000 896628 268873 6378.3744 5522.9069 + 48 16.786543 1000000 896625 267981 6361.5039 5548.21 + 49 17.172406 1000000 896634 267118 6344.7206 5573.3052 + 50 17.554288 1000000 896625 267824 6329.5248 5596.1051 + 51 17.943437 1000000 896630 267082 6314.8587 5618.1251 + 52 18.334562 1000000 896629 266956 6301.5149 5638.1663 + 53 18.730954 1000000 896639 267387 6289.3611 5656.3564 + 54 19.12519 1000000 896635 266585 6276.7946 5675.3065 + 55 19.525844 1000000 896637 266377 6264.414 5693.8682 + 56 19.923464 1000000 896639 267112 6252.0849 5712.4328 + 57 20.325095 1000000 896630 267258 6240.7732 5729.39 + 58 20.717437 1000000 896631 266183 6229.278 5746.6328 + 59 21.112784 1000000 896642 266531 6218.6046 5762.5626 + 60 21.503205 1000000 896641 266076 6210.1826 5775.1171 + 61 21.897496 1000000 896642 266472 6201.1078 5788.7381 + 62 22.297403 1000000 896641 265270 6191.2745 5803.4541 + 63 22.691303 1000000 896645 266277 6182.9482 5815.9763 + 64 23.078438 1000000 896642 265401 6175.3864 5827.3484 + 65 23.479973 1000000 896632 265958 6168.7983 5837.2897 + 66 23.884553 1000000 896637 264949 6162.6626 5846.4277 + 67 24.294635 1000000 896639 265965 6157.0429 5854.905 + 68 24.705267 1000000 896628 266186 6150.5153 5864.7045 + 69 25.119125 1000000 896631 266176 6143.6922 5874.9064 + 70 25.521466 1000000 896626 266155 6139.0928 5881.7854 + 71 25.953995 1000000 896626 265273 6134.0838 5889.3193 + 72 26.357085 1000000 896624 264851 6128.1674 5898.2315 + 73 26.757334 1000000 896628 265570 6122.9351 5906.0567 + 74 27.172586 1000000 896617 265197 6119.2608 5911.6008 + 75 27.577071 1000000 896622 265247 6114.1238 5919.3328 + 76 27.978095 1000000 896632 264691 6109.0948 5926.9208 + 77 28.381971 1000000 896625 264200 6107.3414 5929.5708 + 78 28.788434 1000000 896625 265218 6104.3812 5934.0202 + 79 29.194334 1000000 896624 263872 6100.0506 5940.5028 + 80 29.607791 1000000 896626 264612 6095.7219 5947.0029 + 81 30.009664 1000000 896627 264023 6091.225 5953.8416 + 82 30.40938 1000000 896631 264422 6088.5533 5957.7981 + 83 30.804436 1000000 896627 264917 6085.8755 5961.853 + 84 31.196945 1000000 896628 264811 6084.6025 5963.7579 + 85 31.594372 1000000 896627 265262 6081.8619 5967.8761 + 86 31.982514 1000000 896624 264923 6080.875 5969.3056 + 87 32.379725 1000000 896634 264629 6079.4396 5971.4869 + 88 32.789132 1000000 896634 264138 6078.3155 5973.1707 + 89 33.185249 1000000 896636 265036 6075.2349 5977.8275 + 90 33.569549 1000000 896631 265079 6072.6277 5981.7429 + 91 33.973523 1000000 896636 264351 6069.8461 5985.9147 + 92 34.38997 1000000 896633 264907 6067.4844 5989.4117 + 93 34.804605 1000000 896638 264180 6066.1305 5991.4176 + 94 35.208879 1000000 896627 264728 6064.2879 5994.2068 + 95 35.617876 1000000 896638 264547 6062.9723 5996.1876 + 96 36.029383 1000000 896638 264076 6060.8428 5999.2893 + 97 36.434617 1000000 896636 264284 6060.6075 5999.686 + 98 36.82965 1000000 896634 263609 6058.8001 6002.4136 + 99 37.227715 1000000 896637 265047 6057.6914 6004.1557 + 100 37.618649 1000000 896634 264730 6055.5821 6007.2488 + 101 38.018565 1000000 896644 264497 6053.6048 6010.2635 + 102 38.413535 1000000 896633 264132 6051.6511 6013.1516 + 103 38.81905 1000000 896628 264643 6051.4364 6013.3943 + 104 39.214333 1000000 896637 265039 6049.4288 6016.3891 + 105 39.628715 1000000 896636 264404 6049.7213 6015.981 + 106 40.042658 1000000 896630 264190 6051.0386 6014.037 + 107 40.456148 1000000 896632 263747 6050.0997 6015.4824 + 108 40.871859 1000000 896624 264305 6048.8573 6017.3372 + 109 41.28767 1000000 896623 263860 6049.419 6016.4479 + 110 41.730459 1000000 896624 264242 6049.3865 6016.4998 + 111 42.19708 1000000 896621 265074 6048.8846 6017.206 + 112 42.604722 1000000 896624 264392 6046.1247 6021.3753 + 113 43.000812 1000000 896621 264530 6045.6238 6022.0858 + 114 43.386635 1000000 896625 264673 6045.7114 6021.9866 + 115 43.798834 1000000 896619 264150 6045.2429 6022.7211 + 116 44.202365 1000000 896625 265136 6045.0246 6023.1061 + 117 44.605269 1000000 896627 264300 6044.2782 6024.207 + 118 44.995337 1000000 896624 263827 6043.1335 6025.8787 + 119 45.412376 1000000 896628 263369 6041.4851 6028.3296 + 120 45.827435 1000000 896628 263967 6041.0822 6028.9455 + 121 46.248821 1000000 896619 264658 6042.1182 6027.4477 + 122 46.662872 1000000 896624 263723 6043.0274 6026.0683 + 123 47.081133 1000000 896627 263784 6042.0569 6027.5075 + 124 47.493323 1000000 896634 264026 6042.2947 6027.1131 + 125 47.898065 1000000 896630 265347 6042.3229 6027.0897 + 126 48.296392 1000000 896639 264504 6041.996 6027.6211 + 127 48.694348 1000000 896632 264086 6042.2818 6027.1771 + 128 49.092131 1000000 896634 264613 6041.4223 6028.498 + 129 49.491311 1000000 896635 263841 6039.6019 6031.2108 + 130 49.823591 1000000 896637 264990 6038.1937 6033.2589 + 131 50.1781 1000000 896640 264415 6039.5672 6031.1928 + 132 50.517139 1000000 896644 264120 6039.0161 6032.016 + 133 50.856407 1000000 896644 264990 6037.6454 6034.1162 + 134 51.2213 1000000 896641 263658 6038.1568 6033.3969 + 135 51.589045 1000000 896642 262958 6037.6904 6034.0489 + 136 51.935197 1000000 896634 264809 6037.1925 6034.7615 + 137 52.254541 1000000 896639 264585 6036.4581 6035.8803 + 138 52.642957 1000000 896636 264286 6036.0917 6036.3962 + 139 53.029622 1000000 896633 264202 6036.9673 6035.0906 + 140 53.415754 1000000 896638 263648 6035.0972 6037.9412 + 141 53.803323 1000000 896634 263725 6034.1148 6039.4241 + 142 54.196442 1000000 896628 264045 6032.3751 6042.0888 + 143 54.584199 1000000 896626 264853 6032.2242 6042.2362 + 144 54.968905 1000000 896634 264060 6031.6112 6043.0894 + 145 55.351717 1000000 896635 263927 6031.644 6043.0183 + 146 55.735156 1000000 896634 263760 6031.208 6043.688 + 147 56.125423 1000000 896628 264515 6032.2104 6042.1932 + 148 56.518261 1000000 896629 264685 6033.4865 6040.2827 + 149 56.906283 1000000 896625 264107 6033.8026 6039.7986 + 150 57.295119 1000000 896627 264840 6032.5905 6041.6049 + 151 57.679597 1000000 896625 264098 6033.5935 6040.1017 + 152 58.065542 1000000 896631 264722 6033.7001 6039.9512 + 153 58.449851 1000000 896642 264225 6034.2868 6039.0623 + 154 58.830574 1000000 896631 264458 6033.0212 6041.0402 + 155 59.212543 1000000 896630 264031 6032.4373 6041.9262 + 156 59.594474 1000000 896630 264388 6031.4672 6043.3753 + 157 59.978989 1000000 896629 263860 6031.3996 6043.5387 + 158 60.359462 1000000 896631 264128 6031.5853 6043.2581 + 159 60.745782 1000000 896620 264461 6031.5548 6043.2954 + 160 61.139222 1000000 896623 263834 6031.2224 6043.8271 + 161 61.524795 1000000 896629 264599 6031.3728 6043.5542 + 162 61.91329 1000000 896631 264536 6031.2031 6043.7953 + 163 62.297049 1000000 896628 264026 6032.3531 6042.0958 + 164 62.681785 1000000 896638 263983 6032.4533 6041.8972 + 165 63.066956 1000000 896634 264279 6030.8933 6044.2798 + 166 63.452722 1000000 896635 264741 6029.6235 6046.1894 + 167 63.837621 1000000 896636 263763 6029.4534 6046.4631 + 168 64.224027 1000000 896637 264396 6030.3141 6045.145 + 169 64.611406 1000000 896634 264156 6029.8273 6045.8957 + 170 64.994592 1000000 896631 264489 6030.2754 6045.1968 + 171 65.39128 1000000 896634 263955 6030.9413 6044.2174 + 172 65.783753 1000000 896633 264605 6031.5528 6043.2677 + 173 66.17343 1000000 896631 262956 6029.8457 6045.9212 + 174 66.562387 1000000 896630 264606 6028.509 6047.9159 + 175 66.949714 1000000 896636 264181 6031.1928 6043.9285 + 176 67.33394 1000000 896642 264698 6030.864 6044.3836 + 177 67.718796 1000000 896645 263758 6030.1469 6045.4504 + 178 68.105622 1000000 896644 264581 6029.1587 6046.9095 + 179 68.496908 1000000 896647 264211 6028.5165 6047.936 + 180 68.883654 1000000 896649 264654 6028.5809 6047.747 + 181 69.263973 1000000 896646 264239 6027.1471 6049.9335 + 182 69.5662 1000000 896641 263448 6026.225 6051.3226 + 183 69.908774 1000000 896644 263924 6027.0315 6050.0807 + 184 70.25514 1000000 896641 264640 6026.8847 6050.3142 + 185 70.60226 1000000 896636 263849 6026.7419 6050.5338 + 186 70.949442 1000000 896636 263517 6026.4763 6050.9231 + 187 71.297596 1000000 896637 264417 6026.982 6050.134 + 188 71.632283 1000000 896631 264437 6027.3967 6049.4173 + 189 72.015091 1000000 896631 263554 6029.4067 6046.4633 + 190 72.387086 1000000 896630 263116 6029.3061 6046.6836 + 191 72.756219 1000000 896629 264335 6028.785 6047.4924 + 192 73.044459 1000000 896624 263764 6029.5892 6046.2624 + 193 73.412151 1000000 896627 263888 6029.7553 6046.0985 + 194 73.757971 1000000 896619 264060 6030.8305 6044.3944 + 195 74.070615 1000000 896627 262982 6032.0196 6042.6414 + 196 74.47426 1000000 896626 264220 6033.1426 6040.9646 + 197 74.76225 1000000 896630 264362 6032.2915 6042.2692 + 198 75.087131 1000000 896630 264383 6032.6672 6041.6474 + 199 75.45536 1000000 896634 264276 6032.7147 6041.5822 + 200 75.822295 1000000 896633 263254 6030.618 6044.6613 +Loop time of 75.8223 on 1 procs for 200 steps with 1000000 particles +Performance: 2.638 timesteps/s, 2.638 Mparticle-step/s + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Move | 3.3215 | 3.3215 | 3.3215 | 0.0 | 4.38 +Coll | 68.379 | 68.379 | 68.379 | 0.0 | 90.18 +Sort | 0.93099 | 0.93099 | 0.93099 | 0.0 | 1.23 +Comm | 0.0006318 | 0.0006318 | 0.0006318 | 0.0 | 0.00 +Modify | 0 | 0 | 0 | 0.0 | 0.00 +Output | 3.1897 | 3.1897 | 3.1897 | 0.0 | 4.21 +MPI Sync| 0.00047905 | 0.00047905 | 0.00047905 | 0.0 | 0.00 +Other | | 7.428e-05 | | | 0.00 + +Particle moves = 200000000 (200M) +Cells touched = 213190877 (213M) +Particle comms = 0 (0K) +Boundary collides = 6595278 (6.6M) +Boundary exits = 0 (0K) +SurfColl checks = 0 (0K) +SurfColl occurs = 0 (0K) +Surf reactions = 0 (0K) +Collide attempts = 179326172 (179M) +Collide occurs = 53599802 (53.6M) +Reactions = 0 (0K) +Particles stuck = 0 +Axisymm bad moves = 0 + +Particle-moves/CPUsec/proc: 2.63775e+06 +Particle-moves/step: 1e+06 +Cell-touches/particle/step: 1.06595 +Particle comm iterations/step: 1 +Particle fraction communicated: 0 +Particle fraction colliding with boundary: 0.0329764 +Particle fraction exiting boundary: 0 +Surface-checks/particle/step: 0 +Surface-collisions/particle/step: 0 +Surf-reactions/particle/step: 0 +Collision-attempts/particle/step: 0.896631 +Collisions/particle/step: 0.267999 +Reactions/particle/step: 0 + +Particles: 1e+06 ave 1e+06 max 1e+06 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Cells: 27 ave 27 max 27 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +GhostCell: 0 ave 0 max 0 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +EmptyCell: 0 ave 0 max 0 min +Histogram: 1 0 0 0 0 0 0 0 0 0 diff --git a/examples/relax_variable/log.24Aug26.mpi_4.relax_variable b/examples/relax_variable/log.24Aug26.mpi_4.relax_variable new file mode 100644 index 000000000..4fe73c2ce --- /dev/null +++ b/examples/relax_variable/log.24Aug26.mpi_4.relax_variable @@ -0,0 +1,314 @@ +SPARTA (24 Sep 2025) +Running on 4 MPI task(s) +################################################################################ +# thermal gas in a 3d box with collisions +# particles reflect off global box boundaries +# +# Note: +# - The "comm/sort” option to the “global” command is used to match MPI runs. +# - The “twopass” option is used to match Kokkos runs. +# The "comm/sort" and "twopass" options should not be used for production runs. +################################################################################ + +seed 12345 +dimension 3 +global gridcut 1.0e-5 comm/sort yes + +boundary rr rr rr + +create_box 0 0.0001 0 0.0001 0 0.0001 +Created orthogonal box = (0 0 0) to (0.0001 0.0001 0.0001) +create_grid 3 3 3 +WARNING: Could not acquire nearby ghost cells b/c grid partition is not clumped (/home/runner/work/sparta/sparta/src/grid.cpp:486) +Created 27 child grid cells + CPU time = 0.0014931 secs + create/ghost percent = 91.401 8.59897 + +balance_grid rcb part +Balance grid migrated 24 cells + CPU time = 0.000692408 secs + reassign/sort/migrate/ghost percent = 61.8979 1.46229 11.0923 25.5475 + +species n2.species N2 +mixture air N2 vstream 0.0 0.0 0.0 temp 10000.0 trot 100.0 + +global nrho 7.07043E22 +global fnum 7.07043E5 + +collide vss air n2.vss relax variable + +create_particles air n 1000000 twopass +Created 1000000 particles + CPU time = 0.0713702 secs + +stats 1 +compute temp temp +compute T thermal/grid all all temp +compute Ttrans reduce ave c_T[1] + +compute rot grid all all trot +compute Trot reduce ave c_rot[1] + +stats_style step cpu np nattempt ncoll c_Ttrans c_Trot + +timestep 1.00E-9 +run 200 +Memory usage per proc in Mbytes: + particles (ave,min,max) = 24.2188 21.875 25 + grid (ave,min,max) = 1.51379 1.51379 1.51379 + surf (ave,min,max) = 0 0 0 + modify (ave,min,max) = 0.000514984 0.000457764 0.000534058 + total (ave,min,max) = 25.7331 23.3893 26.5143 +Step CPU Np Natt Ncoll c_Ttrans c_Trot + 0 0 1000000 0 0 9994.2537 100.105 + 1 0.092137964 1000000 896599 300358 9785.4515 413.32821 + 2 0.1844364 1000000 896612 298792 9586.1324 712.25507 + 3 0.27536409 1000000 896619 296541 9399.732 991.85604 + 4 0.36933413 1000000 896619 295631 9221.8866 1258.7513 + 5 0.46409793 1000000 896626 294079 9056.2205 1507.1491 + 6 0.55636314 1000000 896618 292623 8897.1558 1745.8297 + 7 0.64976202 1000000 896622 291230 8748.5976 1968.7432 + 8 0.74492323 1000000 896621 290345 8607.3882 2180.5676 + 9 0.83920425 1000000 896622 288649 8474.3955 2380.0201 + 10 0.93340344 1000000 896622 288621 8348.1751 2569.377 + 11 1.0281653 1000000 896623 286378 8228.1963 2749.386 + 12 1.1223272 1000000 896628 287052 8115.4945 2918.4145 + 13 1.2163449 1000000 896626 284903 8010.5142 3075.8714 + 14 1.3104775 1000000 896626 283421 7909.1366 3227.9504 + 15 1.4045318 1000000 896634 282862 7814.3547 3370.0025 + 16 1.4976463 1000000 896636 282412 7722.5843 3507.5867 + 17 1.5942011 1000000 896634 280925 7635.2754 3638.592 + 18 1.6891788 1000000 896632 280303 7553.5332 3761.2358 + 19 1.78222 1000000 896641 279496 7475.2046 3878.7927 + 20 1.8765 1000000 896632 279202 7401.9971 3988.6457 + 21 1.970558 1000000 896636 278270 7331.9871 4093.7498 + 22 2.064894 1000000 896639 277296 7265.303 4193.7364 + 23 2.1573397 1000000 896639 277395 7200.1947 4291.3711 + 24 2.2516836 1000000 896647 275916 7141.4689 4379.4589 + 25 2.3459428 1000000 896651 276369 7086.1974 4462.4527 + 26 2.4392843 1000000 896650 274642 7032.8629 4542.3363 + 27 2.5333086 1000000 896652 274524 6981.9285 4618.7681 + 28 2.6268649 1000000 896644 274175 6935.3102 4688.6516 + 29 2.7239472 1000000 896645 273456 6889.2451 4757.7546 + 30 2.8174326 1000000 896639 273513 6845.781 4822.9556 + 31 2.9097867 1000000 896646 273056 6805.7205 4883.0317 + 32 3.005304 1000000 896648 272524 6765.3637 4943.5739 + 33 3.1002261 1000000 896641 273195 6729.2635 4997.7037 + 34 3.1934946 1000000 896642 271190 6694.975 5049.1373 + 35 3.2873082 1000000 896649 270840 6663.5701 5096.2146 + 36 3.3823221 1000000 896645 271077 6631.8197 5143.8417 + 37 3.477127 1000000 896635 271166 6602.083 5188.4745 + 38 3.5704667 1000000 896641 270017 6575.737 5227.9603 + 39 3.666321 1000000 896645 269993 6548.982 5268.1075 + 40 3.7594479 1000000 896641 269377 6523.4836 5306.3307 + 41 3.8516491 1000000 896646 269002 6498.9757 5343.1638 + 42 3.9463079 1000000 896654 269953 6475.8309 5377.8488 + 43 4.0382494 1000000 896642 268855 6453.4673 5411.4403 + 44 4.1339334 1000000 896639 268856 6431.9029 5443.7633 + 45 4.2262818 1000000 896641 268046 6413.5223 5471.378 + 46 4.3229209 1000000 896633 268077 6394.0978 5500.5231 + 47 4.4163221 1000000 896644 267893 6375.4599 5528.5195 + 48 4.5106159 1000000 896635 268041 6357.7829 5555.0314 + 49 4.6036838 1000000 896636 267978 6340.6969 5580.6738 + 50 4.6967175 1000000 896637 267322 6324.8101 5604.5165 + 51 4.7923856 1000000 896639 267339 6311.5595 5624.4239 + 52 4.8853404 1000000 896637 266998 6298.2874 5644.3021 + 53 4.9798126 1000000 896630 266443 6284.4269 5665.0356 + 54 5.0736026 1000000 896633 267113 6272.954 5682.2091 + 55 5.1687246 1000000 896637 266477 6260.2742 5701.2232 + 56 5.261143 1000000 896635 267043 6248.4995 5718.9197 + 57 5.3609073 1000000 896638 266628 6238.4142 5734.0794 + 58 5.4533722 1000000 896633 266391 6227.7708 5750.0355 + 59 5.5496582 1000000 896636 265874 6217.518 5765.4187 + 60 5.6428581 1000000 896628 266237 6207.7759 5780.0316 + 61 5.7356703 1000000 896631 265714 6200.2673 5791.317 + 62 5.8301542 1000000 896632 265783 6192.4402 5803.0338 + 63 5.9264891 1000000 896627 265975 6184.2446 5815.3358 + 64 6.0597472 1000000 896626 265767 6175.691 5828.1483 + 65 6.1814482 1000000 896621 266366 6168.8388 5838.456 + 66 6.2917209 1000000 896617 266355 6163.1632 5846.9157 + 67 6.3963531 1000000 896628 265159 6155.6068 5858.2937 + 68 6.5021482 1000000 896627 265405 6150.0281 5866.6819 + 69 6.6059532 1000000 896625 265790 6145.2481 5873.8241 + 70 6.7094569 1000000 896630 265677 6138.8547 5883.3788 + 71 6.8026334 1000000 896625 265082 6132.6886 5892.5882 + 72 6.8963697 1000000 896627 264631 6128.2037 5899.2896 + 73 6.9900566 1000000 896621 265297 6124.6312 5904.7246 + 74 7.0838175 1000000 896627 265169 6121.7529 5909.042 + 75 7.1767071 1000000 896625 265022 6116.3361 5917.1362 + 76 7.2690387 1000000 896624 265888 6112.1402 5923.4532 + 77 7.3631085 1000000 896623 264930 6107.7904 5930.0152 + 78 7.4576423 1000000 896627 264625 6102.4699 5937.976 + 79 7.5518537 1000000 896621 265413 6098.0674 5944.5713 + 80 7.6447609 1000000 896627 264261 6093.5237 5951.3376 + 81 7.7384402 1000000 896621 264448 6090.8486 5955.3704 + 82 7.8356765 1000000 896630 264598 6088.3926 5959.0129 + 83 7.9280259 1000000 896624 265208 6085.3114 5963.6613 + 84 8.0211931 1000000 896626 264399 6084.5905 5964.8175 + 85 8.113554 1000000 896625 264418 6082.3684 5968.1985 + 86 8.2069109 1000000 896626 264634 6079.6892 5972.1722 + 87 8.2998598 1000000 896632 264617 6075.6269 5978.2917 + 88 8.394596 1000000 896632 265725 6074.1384 5980.5779 + 89 8.4891393 1000000 896639 264105 6071.8915 5983.9184 + 90 8.5818076 1000000 896634 264921 6069.8981 5986.8467 + 91 8.7013935 1000000 896630 264968 6068.7819 5988.4803 + 92 8.7951456 1000000 896634 264590 6066.5685 5991.7846 + 93 8.8901557 1000000 896636 264317 6064.4462 5994.9813 + 94 8.9838729 1000000 896632 264993 6060.3535 6001.161 + 95 9.0768799 1000000 896631 263771 6060.2741 6001.2551 + 96 9.1704276 1000000 896632 264125 6059.4115 6002.5929 + 97 9.2640268 1000000 896627 263673 6058.5306 6003.9371 + 98 9.3574873 1000000 896638 264580 6057.5304 6005.4427 + 99 9.449016 1000000 896635 263991 6056.4341 6007.075 + 100 9.5452847 1000000 896630 264873 6056.5198 6006.891 + 101 9.6386434 1000000 896633 265336 6054.0753 6010.503 + 102 9.7319978 1000000 896622 263779 6054.2306 6010.3248 + 103 9.8262094 1000000 896633 264727 6052.2393 6013.2358 + 104 9.9199678 1000000 896623 264193 6050.5577 6015.7187 + 105 10.012926 1000000 896634 263997 6050.5887 6015.7464 + 106 10.105748 1000000 896628 264476 6050.6233 6015.7775 + 107 10.202716 1000000 896632 265621 6049.0522 6018.1618 + 108 10.295745 1000000 896635 264856 6048.7288 6018.6024 + 109 10.390496 1000000 896637 265053 6049.978 6016.7428 + 110 10.485315 1000000 896639 263470 6049.8817 6016.9221 + 111 10.577136 1000000 896640 264024 6047.429 6020.5506 + 112 10.675651 1000000 896638 264227 6048.5098 6018.8973 + 113 10.768716 1000000 896641 264593 6049.2537 6017.7199 + 114 10.861319 1000000 896644 265573 6047.7682 6020.0112 + 115 10.954121 1000000 896640 264342 6047.9246 6019.7518 + 116 11.047274 1000000 896637 264077 6045.2548 6023.8011 + 117 11.140874 1000000 896639 264523 6045.1748 6023.9503 + 118 11.23278 1000000 896639 264554 6045.5358 6023.3889 + 119 11.326197 1000000 896637 264166 6044.1005 6025.6034 + 120 11.418117 1000000 896634 264112 6042.9441 6027.3363 + 121 11.512778 1000000 896635 264197 6044.4981 6024.9514 + 122 11.606462 1000000 896636 264514 6044.5946 6024.8239 + 123 11.700676 1000000 896634 264384 6043.9531 6025.8182 + 124 11.79236 1000000 896631 264137 6043.1691 6026.9851 + 125 11.885549 1000000 896634 263918 6043.5679 6026.3978 + 126 11.979925 1000000 896630 264476 6042.6486 6027.7618 + 127 12.071649 1000000 896631 264562 6041.3816 6029.6723 + 128 12.164439 1000000 896634 264402 6041.4112 6029.633 + 129 12.258688 1000000 896633 264397 6042.7533 6027.5676 + 130 12.35356 1000000 896636 263229 6043.7154 6026.1481 + 131 12.446558 1000000 896634 263607 6043.2435 6026.8694 + 132 12.542069 1000000 896637 264981 6042.7084 6027.6494 + 133 12.634498 1000000 896637 264085 6041.4268 6029.5686 + 134 12.726926 1000000 896637 264507 6039.9377 6031.6991 + 135 12.823523 1000000 896637 263985 6040.4976 6030.8828 + 136 12.915717 1000000 896637 264212 6039.1332 6032.933 + 137 13.008133 1000000 896643 264375 6037.8864 6034.7898 + 138 13.107948 1000000 896640 264798 6036.8171 6036.4541 + 139 13.199504 1000000 896637 264601 6036.5789 6036.8006 + 140 13.293288 1000000 896638 264179 6035.9585 6037.769 + 141 13.386329 1000000 896635 264061 6037.1384 6036.0319 + 142 13.480694 1000000 896631 264936 6037.9201 6034.8659 + 143 13.572285 1000000 896633 264275 6037.6259 6035.3767 + 144 13.665281 1000000 896635 265124 6038.5106 6034.0301 + 145 13.758088 1000000 896628 263811 6038.7848 6033.6057 + 146 13.852009 1000000 896628 265098 6038.4288 6034.0965 + 147 13.944406 1000000 896624 264892 6036.1407 6037.593 + 148 14.037363 1000000 896633 264556 6036.4652 6037.0434 + 149 14.132849 1000000 896628 263744 6036.165 6037.5067 + 150 14.22733 1000000 896636 264680 6036.0125 6037.7622 + 151 14.320776 1000000 896633 264240 6035.2595 6038.9126 + 152 14.413262 1000000 896634 263998 6035.6128 6038.3413 + 153 14.514143 1000000 896635 264215 6036.2369 6037.3385 + 154 14.61296 1000000 896635 264494 6035.5194 6038.4044 + 155 14.714678 1000000 896639 263590 6034.8503 6039.3691 + 156 14.816341 1000000 896631 264131 6033.8154 6040.9569 + 157 14.918389 1000000 896630 263515 6035.2886 6038.8157 + 158 15.015272 1000000 896628 264117 6035.3379 6038.6974 + 159 15.108811 1000000 896632 265117 6035.5976 6038.3035 + 160 15.202081 1000000 896636 264392 6035.8045 6037.9813 + 161 15.301259 1000000 896638 264613 6036.0895 6037.5268 + 162 15.399007 1000000 896639 264229 6036.0503 6037.5742 + 163 15.492158 1000000 896635 264459 6036.2817 6037.277 + 164 15.585558 1000000 896641 264025 6037.3218 6035.7086 + 165 15.679173 1000000 896633 264125 6037.4436 6035.5426 + 166 15.773203 1000000 896633 263880 6037.6133 6035.2668 + 167 15.866161 1000000 896637 264405 6036.9057 6036.394 + 168 15.959028 1000000 896632 264360 6036.9054 6036.3606 + 169 16.052305 1000000 896634 264191 6037.0077 6036.2215 + 170 16.148172 1000000 896629 263506 6038.1079 6034.5831 + 171 16.239895 1000000 896636 264874 6039.162 6032.9443 + 172 16.332045 1000000 896634 263581 6038.258 6034.2849 + 173 16.424826 1000000 896637 263869 6037.6374 6035.195 + 174 16.519483 1000000 896629 264216 6036.9211 6036.2785 + 175 16.612317 1000000 896626 264645 6036.6821 6036.6121 + 176 16.705221 1000000 896623 264046 6035.9274 6037.7159 + 177 16.798823 1000000 896631 264716 6037.3316 6035.6235 + 178 16.891779 1000000 896633 264265 6036.0114 6037.6201 + 179 16.985179 1000000 896631 264334 6035.8392 6037.9101 + 180 17.078817 1000000 896630 263331 6036.428 6037.0523 + 181 17.17265 1000000 896630 264440 6036.1845 6037.3818 + 182 17.267898 1000000 896634 264613 6036.3171 6037.1797 + 183 17.362107 1000000 896624 264530 6038.5615 6033.7891 + 184 17.454945 1000000 896632 264941 6039.105 6032.9923 + 185 17.54993 1000000 896633 264449 6039.6321 6032.2254 + 186 17.643198 1000000 896629 264480 6040.5663 6030.8527 + 187 17.736181 1000000 896636 264693 6040.5342 6030.8767 + 188 17.832154 1000000 896631 264473 6040.7006 6030.6659 + 189 17.926398 1000000 896637 264004 6041.0926 6030.0938 + 190 18.018915 1000000 896634 263334 6040.836 6030.5333 + 191 18.111208 1000000 896633 264734 6041.268 6029.8806 + 192 18.204331 1000000 896629 264735 6040.213 6031.4824 + 193 18.296526 1000000 896638 264086 6040.1412 6031.5697 + 194 18.392122 1000000 896633 263900 6039.913 6031.9223 + 195 18.486501 1000000 896641 265930 6039.4879 6032.5791 + 196 18.581921 1000000 896640 264133 6040.0741 6031.6812 + 197 18.676661 1000000 896628 264182 6040.7139 6030.6992 + 198 18.770611 1000000 896633 264450 6040.9237 6030.3366 + 199 18.864068 1000000 896634 263666 6038.8257 6033.4791 + 200 18.960483 1000000 896633 264694 6038.3179 6034.1679 +Loop time of 18.9607 on 4 procs for 200 steps with 1000000 particles +Performance: 10.548 timesteps/s, 10.548 Mparticle-step/s + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Move | 0.81212 | 0.91456 | 0.94967 | 6.2 | 4.82 +Coll | 13.696 | 15.321 | 15.879 | 24.0 | 80.80 +Sort | 0.60921 | 0.65104 | 0.6708 | 3.1 | 3.43 +Comm | 0.29527 | 0.3008 | 0.30621 | 1.0 | 1.59 +Modify | 0 | 0 | 0 | 0.0 | 0.00 +Output | 1.1428 | 1.7174 | 3.3737 | 73.0 | 9.06 +MPI Sync| 0.0068923 | 0.055824 | 0.16369 | 26.6 | 0.29 +Other | | 6.629e-05 | | | 0.00 + +Particle moves = 200000000 (200M) +Cells touched = 213192135 (213M) +Particle comms = 7618379 (7.62M) +Boundary collides = 6598606 (6.6M) +Boundary exits = 0 (0K) +SurfColl checks = 0 (0K) +SurfColl occurs = 0 (0K) +Surf reactions = 0 (0K) +Collide attempts = 179326628 (179M) +Collide occurs = 53616011 (53.6M) +Reactions = 0 (0K) +Particles stuck = 0 +Axisymm bad moves = 0 + +Particle-moves/CPUsec/proc: 2.63704e+06 +Particle-moves/step: 1e+06 +Cell-touches/particle/step: 1.06596 +Particle comm iterations/step: 1 +Particle fraction communicated: 0.0380919 +Particle fraction colliding with boundary: 0.032993 +Particle fraction exiting boundary: 0 +Surface-checks/particle/step: 0 +Surface-collisions/particle/step: 0 +Surf-reactions/particle/step: 0 +Collision-attempts/particle/step: 0.896633 +Collisions/particle/step: 0.26808 +Reactions/particle/step: 0 + +Particles: 250000 ave 260141 max 221982 min +Histogram: 1 0 0 0 0 0 0 0 0 3 +Cells: 6.75 ave 7 max 6 min +Histogram: 1 0 0 0 0 0 0 0 0 3 +GhostCell: 20.25 ave 21 max 20 min +Histogram: 3 0 0 0 0 0 0 0 0 1 +EmptyCell: 0 ave 0 max 0 min +Histogram: 4 0 0 0 0 0 0 0 0 0 diff --git a/examples/surf_collide/in.beam.adiabatic b/examples/surf_collide/in.beam.adiabatic index ecb33cf3b..2524f9012 100644 --- a/examples/surf_collide/in.beam.adiabatic +++ b/examples/surf_collide/in.beam.adiabatic @@ -29,7 +29,7 @@ surf_collide 1 adiabatic bound_modify zlo collide 1 region circle cylinder z 0 -10 1 INF INF -fix in emit/face/file air zhi data.beam beam_area nevery 100 region circle +fix in emit/face/file air zhi data.beam beam_area nevery 100 region circle twopass #dump 2 image all 10 image.*.ppm type type pdiam 0.2 surf proc 0.01 size 512 512 zoom 1.75 gline no 0.005 #dump_modify 2 pad 4 diff --git a/examples/surf_collide/in.beam.cll b/examples/surf_collide/in.beam.cll index 20c60edc2..99e3a17f1 100644 --- a/examples/surf_collide/in.beam.cll +++ b/examples/surf_collide/in.beam.cll @@ -29,7 +29,7 @@ surf_collide 1 cll 300.0 0.8 0.8 0.8 0.8 #partial 0.5 bound_modify zlo collide 1 region circle cylinder z 0 -10 1 INF INF -fix in emit/face/file air zhi data.beam beam_area nevery 100 region circle +fix in emit/face/file air zhi data.beam beam_area nevery 100 region circle twopass #dump 2 image all 10 image.*.ppm type type pdiam 0.2 surf proc 0.01 size 512 512 zoom 1.75 gline no 0.005 #dump_modify 2 pad 4 diff --git a/examples/surf_collide/in.beam.diffuse b/examples/surf_collide/in.beam.diffuse index e080f15d9..1a6aa5294 100644 --- a/examples/surf_collide/in.beam.diffuse +++ b/examples/surf_collide/in.beam.diffuse @@ -29,7 +29,7 @@ surf_collide 1 diffuse 300 0.5 bound_modify zlo collide 1 region circle cylinder z 0 -10 1 INF INF -fix in emit/face/file air zhi data.beam beam_area nevery 100 region circle +fix in emit/face/file air zhi data.beam beam_area nevery 100 region circle twopass #dump 2 image all 10 image.*.ppm type type pdiam 0.2 surf proc 0.01 size 512 512 zoom 1.75 gline no 0.005 #dump_modify 2 pad 4 diff --git a/examples/surf_collide/in.beam.impulsive b/examples/surf_collide/in.beam.impulsive index 657579875..9fde1c438 100644 --- a/examples/surf_collide/in.beam.impulsive +++ b/examples/surf_collide/in.beam.impulsive @@ -31,7 +31,7 @@ surf_collide 1 impulsive 1000.0 softsphere 0.2 50 2000 60 5 75 #double 10 bound_modify zlo collide 1 region circle cylinder z 0 -10 1 INF INF -fix in emit/face/file air zhi data.beam beam_area nevery 100 region circle +fix in emit/face/file air zhi data.beam beam_area nevery 100 region circle twopass #dump 2 image all 10 image.*.ppm type type pdiam 0.2 surf proc 0.01 size 512 512 zoom 1.75 gline no 0.005 #dump_modify 2 pad 4 diff --git a/examples/surf_collide/in.beam.specular b/examples/surf_collide/in.beam.specular index 6901ddad3..8f70405d0 100644 --- a/examples/surf_collide/in.beam.specular +++ b/examples/surf_collide/in.beam.specular @@ -29,7 +29,7 @@ surf_collide 1 specular bound_modify zlo collide 1 region circle cylinder z 0 -10 1 INF INF -fix in emit/face/file air zhi data.beam beam_area nevery 100 region circle +fix in emit/face/file air zhi data.beam beam_area nevery 100 region circle twopass #dump 2 image all 10 image.*.ppm type type pdiam 0.2 surf proc 0.01 size 512 512 zoom 1.75 gline no 0.005 #dump_modify 2 pad 4 diff --git a/examples/surf_collide/in.beam.td b/examples/surf_collide/in.beam.td index 5a03a7043..c664ec1f2 100644 --- a/examples/surf_collide/in.beam.td +++ b/examples/surf_collide/in.beam.td @@ -33,7 +33,7 @@ surf_collide 1 td 1000.0 #barrier 1000 bound_modify zlo collide 1 region circle cylinder z 0 -10 1 INF INF -fix in emit/face/file air zhi data.beam beam_area nevery 100 region circle +fix in emit/face/file air zhi data.beam beam_area nevery 100 region circle twopass #dump 2 image all 10 image.*.ppm type type pdiam 0.2 surf proc 0.01 size 512 512 zoom 1.75 gline no 0.005 #dump_modify 2 pad 4 diff --git a/examples/surf_collide/in.piston b/examples/surf_collide/in.piston new file mode 100644 index 000000000..b4cde51a6 --- /dev/null +++ b/examples/surf_collide/in.piston @@ -0,0 +1,47 @@ +################################################################################ +# a moving piston compressing a gas, with a vanishing outflow wall +# +# Covers surf_collide piston and surf_collide vanish, neither of which appears +# in any other deck. piston drives xlo inward so the gas is compressed and the +# temperature rises; vanish deletes anything reaching xhi, so Np falls. Both +# effects are visible in stats, which is what makes this a test rather than a +# smoke check. +# +# Note: +# - The "comm/sort" option to the "global" command is used to match MPI runs. +# - The "twopass" option is used to match Kokkos runs. +################################################################################ + +seed 12345 +dimension 3 +global gridcut 1.0e-5 comm/sort yes +boundary s s p +create_box 0 0.0001 0 0.0001 0 0.0001 +create_grid 10 10 10 +balance_grid rcb part + +species air.species N2 N +mixture air N2 N vstream 0.0 0.0 0.0 temp 300.0 +mixture air N2 frac 1.0 +mixture air N frac 0.0 + +global nrho 7.07043E22 fnum 7.07043E6 + +collide vss air air.vss + +surf_collide pist piston 400.0 +surf_collide gone vanish +surf_collide wall specular + +bound_modify xlo collide pist +bound_modify xhi collide gone +bound_modify ylo collide wall +bound_modify yhi collide wall + +create_particles air n 20000 twopass + +compute t temp +stats 50 +stats_style step np nattempt ncoll c_t +timestep 1.0e-9 +run 300 diff --git a/examples/surf_collide/log.11Sep23.mpi_1.beam.adiabatic b/examples/surf_collide/log.22Aug26.mpi_1.beam.adiabatic similarity index 60% rename from examples/surf_collide/log.11Sep23.mpi_1.beam.adiabatic rename to examples/surf_collide/log.22Aug26.mpi_1.beam.adiabatic index c1069e4fc..3474be2a2 100644 --- a/examples/surf_collide/log.11Sep23.mpi_1.beam.adiabatic +++ b/examples/surf_collide/log.22Aug26.mpi_1.beam.adiabatic @@ -1,4 +1,4 @@ -SPARTA (13 Apr 2023) +SPARTA (24 Sep 2025) Running on 1 MPI task(s) ################################################################################ # beam of particles striking the surface at an inclined angle - free molecular flow (no collisions) @@ -19,12 +19,12 @@ create_box -11 11 -11 11 0 10 Created orthogonal box = (-11 -11 0) to (11 11 10) create_grid 5 5 5 Created 125 child grid cells - CPU time = 0.000838901 secs - create/ghost percent = 95.2676 4.73238 + CPU time = 0.00115019 secs + create/ghost percent = 94.1555 5.8445 balance_grid rcb cell Balance grid migrated 0 cells - CPU time = 9.64e-05 secs - reassign/sort/migrate/ghost percent = 66.805 0.414938 16.9087 15.8714 + CPU time = 0.000120939 secs + reassign/sort/migrate/ghost percent = 67.4439 0.725159 10.6318 21.1991 global nrho 1e10 fnum 1e6 @@ -37,8 +37,8 @@ mixture air O frac 0.2 surf_collide 1 adiabatic bound_modify zlo collide 1 -region circle cylinder z 0 -10 1 -INF INF -fix in emit/face/file air zhi data.beam beam_area nevery 100 region circle +region circle cylinder z 0 -10 1 INF INF +fix in emit/face/file air zhi data.beam beam_area nevery 100 region circle twopass #dump 2 image all 10 image.*.ppm type type pdiam 0.2 surf proc 0.01 size 512 512 zoom 1.75 gline no 0.005 #dump_modify 2 pad 4 @@ -52,37 +52,40 @@ Memory usage per proc in Mbytes: particles (ave,min,max) = 0 0 0 grid (ave,min,max) = 1.51379 1.51379 1.51379 surf (ave,min,max) = 0 0 0 + modify (ave,min,max) = 0 0 0 total (ave,min,max) = 1.51379 1.51379 1.51379 Step CPU Np Natt Ncoll Nscoll Nscheck 0 0 0 0 0 0 0 - 100 0.001661804 3118 0 0 0 0 - 200 0.005826412 6225 0 0 0 0 - 300 0.019654539 6933 0 0 0 0 - 400 0.033674766 6968 0 0 0 0 - 500 0.049688897 7007 0 0 0 0 - 600 0.065730628 6951 0 0 0 0 - 700 0.081683259 6983 0 0 0 0 - 800 0.09771129 7037 0 0 0 0 - 900 0.11375852 7068 0 0 0 0 - 1000 0.12960525 7083 0 0 0 0 -Loop time of 0.129657 on 1 procs for 1000 steps with 7083 particles + 100 0.001745897 3117 0 0 0 0 + 200 0.005503405 6221 0 0 0 0 + 300 0.011865047 6928 0 0 0 0 + 400 0.018543761 6948 0 0 0 0 + 500 0.025358365 7004 0 0 0 0 + 600 0.032120899 6936 0 0 0 0 + 700 0.041256694 6979 0 0 0 0 + 800 0.048241666 7054 0 0 0 0 + 900 0.05509878 7067 0 0 0 0 + 1000 0.062118781 7072 0 0 0 0 +Loop time of 0.0621884 on 1 procs for 1000 steps with 7072 particles +Performance: 16080.158 timesteps/s, 113.719 Mparticle-step/s MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Move | 0.056858 | 0.056858 | 0.056858 | 0.0 | 43.85 +Move | 0.053775 | 0.053775 | 0.053775 | 0.0 | 86.47 Coll | 0 | 0 | 0 | 0.0 | 0.00 Sort | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.0002889 | 0.0002889 | 0.0002889 | 0.0 | 0.22 -Modify | 0.0076814 | 0.0076814 | 0.0076814 | 0.0 | 5.92 -Output | 0.064687 | 0.064687 | 0.064687 | 0.0 | 49.89 -Other | | 0.0001418 | | | 0.11 - -Particle moves = 5094446 (5.09M) -Cells touched = 5404464 (5.4M) +Comm | 0.00046288 | 0.00046288 | 0.00046288 | 0.0 | 0.74 +Modify | 0.0071861 | 0.0071861 | 0.0071861 | 0.0 | 11.56 +Output | 0.00048224 | 0.00048224 | 0.00048224 | 0.0 | 0.78 +MPI Sync| 0.00020445 | 0.00020445 | 0.00020445 | 0.0 | 0.33 +Other | | 7.727e-05 | | | 0.12 + +Particle moves = 5092637 (5.09M) +Cells touched = 5402539 (5.4M) Particle comms = 0 (0K) -Boundary collides = 28030 (28K) -Boundary exits = 24087 (24.1K) +Boundary collides = 28019 (28K) +Boundary exits = 24082 (24.1K) SurfColl checks = 0 (0K) SurfColl occurs = 0 (0K) Surf reactions = 0 (0K) @@ -92,13 +95,13 @@ Reactions = 0 (0K) Particles stuck = 0 Axisymm bad moves = 0 -Particle-moves/CPUsec/proc: 3.92919e+07 -Particle-moves/step: 5094.45 +Particle-moves/CPUsec/proc: 8.18904e+07 +Particle-moves/step: 5092.64 Cell-touches/particle/step: 1.06085 Particle comm iterations/step: 1 Particle fraction communicated: 0 -Particle fraction colliding with boundary: 0.00550207 -Particle fraction exiting boundary: 0.00472809 +Particle fraction colliding with boundary: 0.00550186 +Particle fraction exiting boundary: 0.00472879 Surface-checks/particle/step: 0 Surface-collisions/particle/step: 0 Surf-reactions/particle/step: 0 @@ -106,7 +109,7 @@ Collision-attempts/particle/step: 0 Collisions/particle/step: 0 Reactions/particle/step: 0 -Particles: 7083 ave 7083 max 7083 min +Particles: 7072 ave 7072 max 7072 min Histogram: 1 0 0 0 0 0 0 0 0 0 Cells: 125 ave 125 max 125 min Histogram: 1 0 0 0 0 0 0 0 0 0 diff --git a/examples/surf_collide/log.11Sep23.mpi_1.beam.cll b/examples/surf_collide/log.22Aug26.mpi_1.beam.cll similarity index 60% rename from examples/surf_collide/log.11Sep23.mpi_1.beam.cll rename to examples/surf_collide/log.22Aug26.mpi_1.beam.cll index 9f6d0d332..b8936b805 100644 --- a/examples/surf_collide/log.11Sep23.mpi_1.beam.cll +++ b/examples/surf_collide/log.22Aug26.mpi_1.beam.cll @@ -1,4 +1,4 @@ -SPARTA (13 Apr 2023) +SPARTA (24 Sep 2025) Running on 1 MPI task(s) ################################################################################ # beam of particles striking the surface at an inclined angle - free molecular flow (no collisions) @@ -19,12 +19,12 @@ create_box -11 11 -11 11 0 10 Created orthogonal box = (-11 -11 0) to (11 11 10) create_grid 5 5 5 Created 125 child grid cells - CPU time = 0.000844401 secs - create/ghost percent = 95.4169 4.58313 + CPU time = 0.00121177 secs + create/ghost percent = 95.0006 4.99937 balance_grid rcb cell Balance grid migrated 0 cells - CPU time = 9.8001e-05 secs - reassign/sort/migrate/ghost percent = 66.8371 0.408159 16.9386 15.8162 + CPU time = 0.00014787 secs + reassign/sort/migrate/ghost percent = 73.5836 0.499087 8.43714 17.4802 global nrho 1e10 fnum 1e6 @@ -37,8 +37,8 @@ mixture air O frac 0.2 surf_collide 1 cll 300.0 0.8 0.8 0.8 0.8 #partial 0.5 bound_modify zlo collide 1 -region circle cylinder z 0 -10 1 -INF INF -fix in emit/face/file air zhi data.beam beam_area nevery 100 region circle +region circle cylinder z 0 -10 1 INF INF +fix in emit/face/file air zhi data.beam beam_area nevery 100 region circle twopass #dump 2 image all 10 image.*.ppm type type pdiam 0.2 surf proc 0.01 size 512 512 zoom 1.75 gline no 0.005 #dump_modify 2 pad 4 @@ -52,37 +52,40 @@ Memory usage per proc in Mbytes: particles (ave,min,max) = 0 0 0 grid (ave,min,max) = 1.51379 1.51379 1.51379 surf (ave,min,max) = 0 0 0 + modify (ave,min,max) = 0 0 0 total (ave,min,max) = 1.51379 1.51379 1.51379 Step CPU Np Natt Ncoll Nscoll Nscheck 0 0 0 0 0 0 0 - 100 0.001666804 3118 0 0 0 0 - 200 0.014133628 6225 0 0 0 0 - 300 0.021722143 8790 0 0 0 0 - 400 0.0511067 9411 0 0 0 0 - 500 0.071443239 9577 0 0 0 0 - 600 0.097278689 9614 0 0 0 0 - 700 0.11517012 9693 0 0 0 0 - 800 0.13300696 9682 0 0 0 0 - 900 0.16766663 9746 0 0 0 0 - 1000 0.19054447 9743 0 0 0 0 -Loop time of 0.190565 on 1 procs for 1000 steps with 9743 particles + 100 0.001734715 3117 0 0 0 0 + 200 0.00550651 6221 0 0 0 0 + 300 0.012384996 8772 0 0 0 0 + 400 0.020367999 9385 0 0 0 0 + 500 0.028776563 9562 0 0 0 0 + 600 0.037466745 9620 0 0 0 0 + 700 0.04598842 9686 0 0 0 0 + 800 0.054672702 9667 0 0 0 0 + 900 0.063176777 9740 0 0 0 0 + 1000 0.071750129 9725 0 0 0 0 +Loop time of 0.0717909 on 1 procs for 1000 steps with 9725 particles +Performance: 13929.333 timesteps/s, 135.463 Mparticle-step/s MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Move | 0.14192 | 0.14192 | 0.14192 | 0.0 | 74.47 +Move | 0.06343 | 0.06343 | 0.06343 | 0.0 | 88.35 Coll | 0 | 0 | 0 | 0.0 | 0.00 Sort | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.0002889 | 0.0002889 | 0.0002889 | 0.0 | 0.15 -Modify | 0.02409 | 0.02409 | 0.02409 | 0.0 | 12.64 -Output | 0.024116 | 0.024116 | 0.024116 | 0.0 | 12.66 -Other | | 0.0001512 | | | 0.08 - -Particle moves = 6537279 (6.54M) -Cells touched = 6842678 (6.84M) +Comm | 0.00046466 | 0.00046466 | 0.00046466 | 0.0 | 0.65 +Modify | 0.0071779 | 0.0071779 | 0.0071779 | 0.0 | 10.00 +Output | 0.00047444 | 0.00047444 | 0.00047444 | 0.0 | 0.66 +MPI Sync| 0.00016946 | 0.00016946 | 0.00016946 | 0.0 | 0.24 +Other | | 7.446e-05 | | | 0.10 + +Particle moves = 6533852 (6.53M) +Cells touched = 6839083 (6.84M) Particle comms = 0 (0K) -Boundary collides = 28030 (28K) -Boundary exits = 21427 (21.4K) +Boundary collides = 28019 (28K) +Boundary exits = 21429 (21.4K) SurfColl checks = 0 (0K) SurfColl occurs = 0 (0K) Surf reactions = 0 (0K) @@ -92,13 +95,13 @@ Reactions = 0 (0K) Particles stuck = 0 Axisymm bad moves = 0 -Particle-moves/CPUsec/proc: 3.43046e+07 -Particle-moves/step: 6537.28 +Particle-moves/CPUsec/proc: 9.10122e+07 +Particle-moves/step: 6533.85 Cell-touches/particle/step: 1.04672 Particle comm iterations/step: 1 Particle fraction communicated: 0 -Particle fraction colliding with boundary: 0.00428772 -Particle fraction exiting boundary: 0.00327766 +Particle fraction colliding with boundary: 0.00428828 +Particle fraction exiting boundary: 0.00327969 Surface-checks/particle/step: 0 Surface-collisions/particle/step: 0 Surf-reactions/particle/step: 0 @@ -106,7 +109,7 @@ Collision-attempts/particle/step: 0 Collisions/particle/step: 0 Reactions/particle/step: 0 -Particles: 9743 ave 9743 max 9743 min +Particles: 9725 ave 9725 max 9725 min Histogram: 1 0 0 0 0 0 0 0 0 0 Cells: 125 ave 125 max 125 min Histogram: 1 0 0 0 0 0 0 0 0 0 diff --git a/examples/surf_collide/log.11Sep23.mpi_1.beam.diffuse b/examples/surf_collide/log.22Aug26.mpi_1.beam.diffuse similarity index 59% rename from examples/surf_collide/log.11Sep23.mpi_1.beam.diffuse rename to examples/surf_collide/log.22Aug26.mpi_1.beam.diffuse index 08317a108..b9185a9b3 100644 --- a/examples/surf_collide/log.11Sep23.mpi_1.beam.diffuse +++ b/examples/surf_collide/log.22Aug26.mpi_1.beam.diffuse @@ -1,4 +1,4 @@ -SPARTA (13 Apr 2023) +SPARTA (24 Sep 2025) Running on 1 MPI task(s) ################################################################################ # beam of particles striking the surface at an inclined angle - free molecular flow (no collisions) @@ -19,12 +19,12 @@ create_box -11 11 -11 11 0 10 Created orthogonal box = (-11 -11 0) to (11 11 10) create_grid 5 5 5 Created 125 child grid cells - CPU time = 0.000874402 secs - create/ghost percent = 95.5856 4.41445 + CPU time = 0.00117358 secs + create/ghost percent = 94.4591 5.54093 balance_grid rcb cell Balance grid migrated 0 cells - CPU time = 9.6801e-05 secs - reassign/sort/migrate/ghost percent = 67.7689 0.309914 15.8056 16.1155 + CPU time = 0.000120252 secs + reassign/sort/migrate/ghost percent = 67.8093 0.559658 10.1778 21.4533 global nrho 1e10 fnum 1e6 @@ -37,8 +37,8 @@ mixture air O frac 0.2 surf_collide 1 diffuse 300 0.5 bound_modify zlo collide 1 -region circle cylinder z 0 -10 1 -INF INF -fix in emit/face/file air zhi data.beam beam_area nevery 100 region circle +region circle cylinder z 0 -10 1 INF INF +fix in emit/face/file air zhi data.beam beam_area nevery 100 region circle twopass #dump 2 image all 10 image.*.ppm type type pdiam 0.2 surf proc 0.01 size 512 512 zoom 1.75 gline no 0.005 #dump_modify 2 pad 4 @@ -52,37 +52,40 @@ Memory usage per proc in Mbytes: particles (ave,min,max) = 0 0 0 grid (ave,min,max) = 1.51379 1.51379 1.51379 surf (ave,min,max) = 0 0 0 + modify (ave,min,max) = 0 0 0 total (ave,min,max) = 1.51379 1.51379 1.51379 Step CPU Np Natt Ncoll Nscoll Nscheck 0 0 0 0 0 0 0 - 100 0.001633204 3118 0 0 0 0 - 200 0.005805014 6225 0 0 0 0 - 300 0.022343053 7689 0 0 0 0 - 400 0.047483112 8262 0 0 0 0 - 500 0.064496252 8393 0 0 0 0 - 600 0.081655292 8428 0 0 0 0 - 700 0.090806513 8419 0 0 0 0 - 800 0.11599557 8515 0 0 0 0 - 900 0.13330241 8649 0 0 0 0 - 1000 0.15234586 8591 0 0 0 0 -Loop time of 0.152414 on 1 procs for 1000 steps with 8591 particles + 100 0.00209062 3117 0 0 0 0 + 200 0.00581702 6221 0 0 0 0 + 300 0.0125362 7688 0 0 0 0 + 400 0.020482375 8253 0 0 0 0 + 500 0.028454732 8390 0 0 0 0 + 600 0.039584352 8419 0 0 0 0 + 700 0.047570925 8436 0 0 0 0 + 800 0.055811745 8515 0 0 0 0 + 900 0.063948625 8643 0 0 0 0 + 1000 0.072246323 8562 0 0 0 0 +Loop time of 0.0722891 on 1 procs for 1000 steps with 8562 particles +Performance: 13833.343 timesteps/s, 118.441 Mparticle-step/s MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Move | 0.092733 | 0.092733 | 0.092733 | 0.0 | 60.84 +Move | 0.063422 | 0.063422 | 0.063422 | 0.0 | 87.73 Coll | 0 | 0 | 0 | 0.0 | 0.00 Sort | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.000378 | 0.000378 | 0.000378 | 0.0 | 0.25 -Modify | 0.023632 | 0.023632 | 0.023632 | 0.0 | 15.50 -Output | 0.035526 | 0.035526 | 0.035526 | 0.0 | 23.31 -Other | | 0.0001453 | | | 0.10 - -Particle moves = 6265625 (6.27M) -Cells touched = 6574816 (6.57M) +Comm | 0.00063056 | 0.00063056 | 0.00063056 | 0.0 | 0.87 +Modify | 0.0075944 | 0.0075944 | 0.0075944 | 0.0 | 10.51 +Output | 0.0004007 | 0.0004007 | 0.0004007 | 0.0 | 0.55 +MPI Sync| 0.00017645 | 0.00017645 | 0.00017645 | 0.0 | 0.24 +Other | | 6.538e-05 | | | 0.09 + +Particle moves = 6262684 (6.26M) +Cells touched = 6571823 (6.57M) Particle comms = 0 (0K) -Boundary collides = 28030 (28K) -Boundary exits = 22579 (22.6K) +Boundary collides = 28019 (28K) +Boundary exits = 22592 (22.6K) SurfColl checks = 0 (0K) SurfColl occurs = 0 (0K) Surf reactions = 0 (0K) @@ -92,13 +95,13 @@ Reactions = 0 (0K) Particles stuck = 0 Axisymm bad moves = 0 -Particle-moves/CPUsec/proc: 4.11092e+07 -Particle-moves/step: 6265.62 -Cell-touches/particle/step: 1.04935 +Particle-moves/CPUsec/proc: 8.66339e+07 +Particle-moves/step: 6262.68 +Cell-touches/particle/step: 1.04936 Particle comm iterations/step: 1 Particle fraction communicated: 0 -Particle fraction colliding with boundary: 0.00447362 -Particle fraction exiting boundary: 0.00360363 +Particle fraction colliding with boundary: 0.00447396 +Particle fraction exiting boundary: 0.0036074 Surface-checks/particle/step: 0 Surface-collisions/particle/step: 0 Surf-reactions/particle/step: 0 @@ -106,7 +109,7 @@ Collision-attempts/particle/step: 0 Collisions/particle/step: 0 Reactions/particle/step: 0 -Particles: 8591 ave 8591 max 8591 min +Particles: 8562 ave 8562 max 8562 min Histogram: 1 0 0 0 0 0 0 0 0 0 Cells: 125 ave 125 max 125 min Histogram: 1 0 0 0 0 0 0 0 0 0 diff --git a/examples/surf_collide/log.11Sep23.mpi_1.beam.impulsive b/examples/surf_collide/log.22Aug26.mpi_1.beam.impulsive similarity index 61% rename from examples/surf_collide/log.11Sep23.mpi_1.beam.impulsive rename to examples/surf_collide/log.22Aug26.mpi_1.beam.impulsive index d46638863..fbaac02d1 100644 --- a/examples/surf_collide/log.11Sep23.mpi_1.beam.impulsive +++ b/examples/surf_collide/log.22Aug26.mpi_1.beam.impulsive @@ -1,4 +1,4 @@ -SPARTA (13 Apr 2023) +SPARTA (24 Sep 2025) Running on 1 MPI task(s) ################################################################################ # beam of particles striking the surface at an inclined angle - free molecular flow (no collisions) @@ -19,12 +19,12 @@ create_box -11 11 -11 11 0 10 Created orthogonal box = (-11 -11 0) to (11 11 10) create_grid 1 1 1 Created 1 child grid cells - CPU time = 0.000827602 secs - create/ghost percent = 98.3325 1.66747 + CPU time = 0.00110637 secs + create/ghost percent = 97.4989 2.50105 balance_grid rcb cell Balance grid migrated 0 cells - CPU time = 8.2401e-05 secs - reassign/sort/migrate/ghost percent = 79.4893 0.364073 15.7777 4.36888 + CPU time = 8.6754e-05 secs + reassign/sort/migrate/ghost percent = 83.059 0.569426 12.0006 4.37098 global nrho 1e10 fnum 1e6 @@ -39,8 +39,8 @@ surf_collide 1 impulsive 1000.0 softsphere 0.2 50 2000 60 5 75 #double 10 bound_modify zlo collide 1 -region circle cylinder z 0 -10 1 -INF INF -fix in emit/face/file air zhi data.beam beam_area nevery 100 region circle +region circle cylinder z 0 -10 1 INF INF +fix in emit/face/file air zhi data.beam beam_area nevery 100 region circle twopass #dump 2 image all 10 image.*.ppm type type pdiam 0.2 surf proc 0.01 size 512 512 zoom 1.75 gline no 0.005 #dump_modify 2 pad 4 @@ -54,37 +54,40 @@ Memory usage per proc in Mbytes: particles (ave,min,max) = 0 0 0 grid (ave,min,max) = 1.51379 1.51379 1.51379 surf (ave,min,max) = 0 0 0 + modify (ave,min,max) = 0 0 0 total (ave,min,max) = 1.51379 1.51379 1.51379 Step CPU Np Natt Ncoll Nscoll Nscheck 0 0 0 0 0 0 0 - 100 0.001627504 3118 0 0 0 0 - 200 0.017415641 6225 0 0 0 0 - 300 0.039480493 7260 0 0 0 0 - 400 0.058024836 7486 0 0 0 0 - 500 0.081526891 7595 0 0 0 0 - 600 0.10254434 7584 0 0 0 0 - 700 0.12501389 7547 0 0 0 0 - 800 0.14905035 7578 0 0 0 0 - 900 0.1718553 7668 0 0 0 0 - 1000 0.19079015 7656 0 0 0 0 -Loop time of 0.198292 on 1 procs for 1000 steps with 7656 particles + 100 0.001739866 3117 0 0 0 0 + 200 0.008760106 6221 0 0 0 0 + 300 0.017827564 7246 0 0 0 0 + 400 0.027018971 7492 0 0 0 0 + 500 0.036598271 7560 0 0 0 0 + 600 0.045982044 7559 0 0 0 0 + 700 0.055257388 7546 0 0 0 0 + 800 0.064798758 7609 0 0 0 0 + 900 0.074247453 7697 0 0 0 0 + 1000 0.084018773 7639 0 0 0 0 +Loop time of 0.084061 on 1 procs for 1000 steps with 7639 particles +Performance: 11896.121 timesteps/s, 90.874 Mparticle-step/s MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Move | 0.15283 | 0.15283 | 0.15283 | 0.0 | 77.07 +Move | 0.076061 | 0.076061 | 0.076061 | 0.0 | 90.48 Coll | 0 | 0 | 0 | 0.0 | 0.00 Sort | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.0003088 | 0.0003088 | 0.0003088 | 0.0 | 0.16 -Modify | 0.015528 | 0.015528 | 0.015528 | 0.0 | 7.83 -Output | 0.029478 | 0.029478 | 0.029478 | 0.0 | 14.87 -Other | | 0.0001471 | | | 0.07 - -Particle moves = 5231166 (5.23M) -Cells touched = 5231166 (5.23M) +Comm | 0.00034875 | 0.00034875 | 0.00034875 | 0.0 | 0.41 +Modify | 0.0070615 | 0.0070615 | 0.0070615 | 0.0 | 8.40 +Output | 0.00037306 | 0.00037306 | 0.00037306 | 0.0 | 0.44 +MPI Sync| 0.00015561 | 0.00015561 | 0.00015561 | 0.0 | 0.19 +Other | | 6.128e-05 | | | 0.07 + +Particle moves = 5231445 (5.23M) +Cells touched = 5231445 (5.23M) Particle comms = 0 (0K) -Boundary collides = 28030 (28K) -Boundary exits = 23514 (23.5K) +Boundary collides = 28019 (28K) +Boundary exits = 23515 (23.5K) SurfColl checks = 0 (0K) SurfColl occurs = 0 (0K) Surf reactions = 0 (0K) @@ -94,13 +97,13 @@ Reactions = 0 (0K) Particles stuck = 0 Axisymm bad moves = 0 -Particle-moves/CPUsec/proc: 2.63812e+07 -Particle-moves/step: 5231.17 +Particle-moves/CPUsec/proc: 6.22339e+07 +Particle-moves/step: 5231.44 Cell-touches/particle/step: 1 Particle comm iterations/step: 1 Particle fraction communicated: 0 -Particle fraction colliding with boundary: 0.00535827 -Particle fraction exiting boundary: 0.00449498 +Particle fraction colliding with boundary: 0.00535588 +Particle fraction exiting boundary: 0.00449493 Surface-checks/particle/step: 0 Surface-collisions/particle/step: 0 Surf-reactions/particle/step: 0 @@ -108,7 +111,7 @@ Collision-attempts/particle/step: 0 Collisions/particle/step: 0 Reactions/particle/step: 0 -Particles: 7656 ave 7656 max 7656 min +Particles: 7639 ave 7639 max 7639 min Histogram: 1 0 0 0 0 0 0 0 0 0 Cells: 1 ave 1 max 1 min Histogram: 1 0 0 0 0 0 0 0 0 0 diff --git a/examples/surf_collide/log.11Sep23.mpi_1.beam.specular b/examples/surf_collide/log.22Aug26.mpi_1.beam.specular similarity index 60% rename from examples/surf_collide/log.11Sep23.mpi_1.beam.specular rename to examples/surf_collide/log.22Aug26.mpi_1.beam.specular index f03b22272..5d9b95199 100644 --- a/examples/surf_collide/log.11Sep23.mpi_1.beam.specular +++ b/examples/surf_collide/log.22Aug26.mpi_1.beam.specular @@ -1,4 +1,4 @@ -SPARTA (13 Apr 2023) +SPARTA (24 Sep 2025) Running on 1 MPI task(s) ################################################################################ # beam of particles striking the surface at an inclined angle - free molecular flow (no collisions) @@ -19,12 +19,12 @@ create_box -11 11 -11 11 0 10 Created orthogonal box = (-11 -11 0) to (11 11 10) create_grid 5 5 5 Created 125 child grid cells - CPU time = 0.000850102 secs - create/ghost percent = 95.0594 4.94058 + CPU time = 0.0011415 secs + create/ghost percent = 94.1666 5.83336 balance_grid rcb cell Balance grid migrated 0 cells - CPU time = 9.6001e-05 secs - reassign/sort/migrate/ghost percent = 68.2295 0.416662 15.2082 16.1457 + CPU time = 0.000119567 secs + reassign/sort/migrate/ghost percent = 67.7645 0.556174 10.7379 20.9414 global nrho 1e10 fnum 1e6 @@ -37,8 +37,8 @@ mixture air O frac 0.2 surf_collide 1 specular bound_modify zlo collide 1 -region circle cylinder z 0 -10 1 -INF INF -fix in emit/face/file air zhi data.beam beam_area nevery 100 region circle +region circle cylinder z 0 -10 1 INF INF +fix in emit/face/file air zhi data.beam beam_area nevery 100 region circle twopass #dump 2 image all 10 image.*.ppm type type pdiam 0.2 surf proc 0.01 size 512 512 zoom 1.75 gline no 0.005 #dump_modify 2 pad 4 @@ -52,37 +52,40 @@ Memory usage per proc in Mbytes: particles (ave,min,max) = 0 0 0 grid (ave,min,max) = 1.51379 1.51379 1.51379 surf (ave,min,max) = 0 0 0 + modify (ave,min,max) = 0 0 0 total (ave,min,max) = 1.51379 1.51379 1.51379 Step CPU Np Natt Ncoll Nscoll Nscheck 0 0 0 0 0 0 0 - 100 0.001634904 3101 0 0 0 0 - 200 0.00842932 6270 0 0 0 0 - 300 0.021566151 6382 0 0 0 0 - 400 0.037606088 6354 0 0 0 0 - 500 0.053616426 6338 0 0 0 0 - 600 0.060830143 6334 0 0 0 0 - 700 0.077585382 6389 0 0 0 0 - 800 0.09354792 6412 0 0 0 0 - 900 0.10959896 6359 0 0 0 0 - 1000 0.12554579 6335 0 0 0 0 -Loop time of 0.134319 on 1 procs for 1000 steps with 6335 particles + 100 0.001759458 3098 0 0 0 0 + 200 0.005190917 6270 0 0 0 0 + 300 0.011592884 6383 0 0 0 0 + 400 0.018383521 6349 0 0 0 0 + 500 0.024847073 6335 0 0 0 0 + 600 0.031421334 6334 0 0 0 0 + 700 0.037703662 6393 0 0 0 0 + 800 0.044009265 6414 0 0 0 0 + 900 0.050572726 6355 0 0 0 0 + 1000 0.057230263 6334 0 0 0 0 +Loop time of 0.0572774 on 1 procs for 1000 steps with 6334 particles +Performance: 17458.901 timesteps/s, 110.585 Mparticle-step/s MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Move | 0.05815 | 0.05815 | 0.05815 | 0.0 | 43.29 +Move | 0.04853 | 0.04853 | 0.04853 | 0.0 | 84.73 Coll | 0 | 0 | 0 | 0.0 | 0.00 Sort | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.0004157 | 0.0004157 | 0.0004157 | 0.0 | 0.31 -Modify | 0.0076304 | 0.0076304 | 0.0076304 | 0.0 | 5.68 -Output | 0.06798 | 0.06798 | 0.06798 | 0.0 | 50.61 -Other | | 0.0001424 | | | 0.11 - -Particle moves = 5383479 (5.38M) -Cells touched = 5704605 (5.7M) +Comm | 0.00051671 | 0.00051671 | 0.00051671 | 0.0 | 0.90 +Modify | 0.007555 | 0.007555 | 0.007555 | 0.0 | 13.19 +Output | 0.00046288 | 0.00046288 | 0.00046288 | 0.0 | 0.81 +MPI Sync| 0.00015455 | 0.00015455 | 0.00015455 | 0.0 | 0.27 +Other | | 5.802e-05 | | | 0.10 + +Particle moves = 5382677 (5.38M) +Cells touched = 5703755 (5.7M) Particle comms = 0 (0K) -Boundary collides = 28312 (28.3K) -Boundary exits = 25108 (25.1K) +Boundary collides = 28306 (28.3K) +Boundary exits = 25105 (25.1K) SurfColl checks = 0 (0K) SurfColl occurs = 0 (0K) Surf reactions = 0 (0K) @@ -92,13 +95,13 @@ Reactions = 0 (0K) Particles stuck = 0 Axisymm bad moves = 0 -Particle-moves/CPUsec/proc: 4.00798e+07 -Particle-moves/step: 5383.48 +Particle-moves/CPUsec/proc: 9.39756e+07 +Particle-moves/step: 5382.68 Cell-touches/particle/step: 1.05965 Particle comm iterations/step: 1 Particle fraction communicated: 0 -Particle fraction colliding with boundary: 0.00525905 -Particle fraction exiting boundary: 0.0046639 +Particle fraction colliding with boundary: 0.00525872 +Particle fraction exiting boundary: 0.00466404 Surface-checks/particle/step: 0 Surface-collisions/particle/step: 0 Surf-reactions/particle/step: 0 @@ -106,7 +109,7 @@ Collision-attempts/particle/step: 0 Collisions/particle/step: 0 Reactions/particle/step: 0 -Particles: 6335 ave 6335 max 6335 min +Particles: 6334 ave 6334 max 6334 min Histogram: 1 0 0 0 0 0 0 0 0 0 Cells: 125 ave 125 max 125 min Histogram: 1 0 0 0 0 0 0 0 0 0 diff --git a/examples/surf_collide/log.11Sep23.mpi_1.beam.td b/examples/surf_collide/log.22Aug26.mpi_1.beam.td similarity index 62% rename from examples/surf_collide/log.11Sep23.mpi_1.beam.td rename to examples/surf_collide/log.22Aug26.mpi_1.beam.td index 20cea59b0..64a6a4723 100644 --- a/examples/surf_collide/log.11Sep23.mpi_1.beam.td +++ b/examples/surf_collide/log.22Aug26.mpi_1.beam.td @@ -1,4 +1,4 @@ -SPARTA (13 Apr 2023) +SPARTA (24 Sep 2025) Running on 1 MPI task(s) ################################################################################ # beam of particles striking the surface at an inclined angle - free molecular flow (no collisions) @@ -19,12 +19,12 @@ create_box -11 11 -11 11 0 10 Created orthogonal box = (-11 -11 0) to (11 11 10) create_grid 1 1 1 Created 1 child grid cells - CPU time = 0.000848402 secs - create/ghost percent = 98.2673 1.73267 + CPU time = 0.00116095 secs + create/ghost percent = 97.3904 2.60958 balance_grid rcb cell Balance grid migrated 0 cells - CPU time = 7.56e-05 secs - reassign/sort/migrate/ghost percent = 77.6455 0.26455 17.5926 4.49735 + CPU time = 8.9791e-05 secs + reassign/sort/migrate/ghost percent = 83.2366 0.616988 12.3442 3.80216 global nrho 1e10 fnum 1e6 @@ -41,8 +41,8 @@ surf_collide 1 td 1000.0 #barrier 1000 bound_modify zlo collide 1 -region circle cylinder z 0 -10 1 -INF INF -fix in emit/face/file air zhi data.beam beam_area nevery 100 region circle +region circle cylinder z 0 -10 1 INF INF +fix in emit/face/file air zhi data.beam beam_area nevery 100 region circle twopass #dump 2 image all 10 image.*.ppm type type pdiam 0.2 surf proc 0.01 size 512 512 zoom 1.75 gline no 0.005 #dump_modify 2 pad 4 @@ -56,37 +56,40 @@ Memory usage per proc in Mbytes: particles (ave,min,max) = 0 0 0 grid (ave,min,max) = 1.51379 1.51379 1.51379 surf (ave,min,max) = 0 0 0 + modify (ave,min,max) = 0 0 0 total (ave,min,max) = 1.51379 1.51379 1.51379 Step CPU Np Natt Ncoll Nscoll Nscheck 0 0 0 0 0 0 0 - 100 0.001619503 3118 0 0 0 0 - 200 0.005599612 6225 0 0 0 0 - 300 0.021728248 7461 0 0 0 0 - 400 0.038177485 7628 0 0 0 0 - 500 0.045212501 7682 0 0 0 0 - 600 0.066404748 7701 0 0 0 0 - 700 0.073511364 7763 0 0 0 0 - 800 0.090454501 7772 0 0 0 0 - 900 0.10640754 7847 0 0 0 0 - 1000 0.11452185 7765 0 0 0 0 -Loop time of 0.123312 on 1 procs for 1000 steps with 7765 particles + 100 0.001741964 3117 0 0 0 0 + 200 0.005326963 6221 0 0 0 0 + 300 0.011394349 7450 0 0 0 0 + 400 0.017422236 7621 0 0 0 0 + 500 0.023852657 7669 0 0 0 0 + 600 0.029984826 7699 0 0 0 0 + 700 0.036910661 7768 0 0 0 0 + 800 0.045463652 7777 0 0 0 0 + 900 0.051731613 7841 0 0 0 0 + 1000 0.057919468 7783 0 0 0 0 +Loop time of 0.0579685 on 1 procs for 1000 steps with 7783 particles +Performance: 17250.749 timesteps/s, 134.263 Mparticle-step/s MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Move | 0.061365 | 0.061365 | 0.061365 | 0.0 | 49.76 +Move | 0.049849 | 0.049849 | 0.049849 | 0.0 | 85.99 Coll | 0 | 0 | 0 | 0.0 | 0.00 Sort | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.0002913 | 0.0002913 | 0.0002913 | 0.0 | 0.24 -Modify | 0.0075101 | 0.0075101 | 0.0075101 | 0.0 | 6.09 -Output | 0.054006 | 0.054006 | 0.054006 | 0.0 | 43.80 -Other | | 0.0001386 | | | 0.11 - -Particle moves = 5391255 (5.39M) -Cells touched = 5391255 (5.39M) +Comm | 0.00035908 | 0.00035908 | 0.00035908 | 0.0 | 0.62 +Modify | 0.0071083 | 0.0071083 | 0.0071083 | 0.0 | 12.26 +Output | 0.00038233 | 0.00038233 | 0.00038233 | 0.0 | 0.66 +MPI Sync| 0.00018704 | 0.00018704 | 0.00018704 | 0.0 | 0.32 +Other | | 8.251e-05 | | | 0.14 + +Particle moves = 5389397 (5.39M) +Cells touched = 5389397 (5.39M) Particle comms = 0 (0K) -Boundary collides = 28030 (28K) -Boundary exits = 23405 (23.4K) +Boundary collides = 28019 (28K) +Boundary exits = 23371 (23.4K) SurfColl checks = 0 (0K) SurfColl occurs = 0 (0K) Surf reactions = 0 (0K) @@ -96,13 +99,13 @@ Reactions = 0 (0K) Particles stuck = 0 Axisymm bad moves = 0 -Particle-moves/CPUsec/proc: 4.37206e+07 -Particle-moves/step: 5391.26 +Particle-moves/CPUsec/proc: 9.29711e+07 +Particle-moves/step: 5389.4 Cell-touches/particle/step: 1 Particle comm iterations/step: 1 Particle fraction communicated: 0 -Particle fraction colliding with boundary: 0.00519916 -Particle fraction exiting boundary: 0.00434129 +Particle fraction colliding with boundary: 0.00519891 +Particle fraction exiting boundary: 0.00433648 Surface-checks/particle/step: 0 Surface-collisions/particle/step: 0 Surf-reactions/particle/step: 0 @@ -110,7 +113,7 @@ Collision-attempts/particle/step: 0 Collisions/particle/step: 0 Reactions/particle/step: 0 -Particles: 7765 ave 7765 max 7765 min +Particles: 7783 ave 7783 max 7783 min Histogram: 1 0 0 0 0 0 0 0 0 0 Cells: 1 ave 1 max 1 min Histogram: 1 0 0 0 0 0 0 0 0 0 diff --git a/examples/surf_collide/log.22Aug26.mpi_1.piston b/examples/surf_collide/log.22Aug26.mpi_1.piston new file mode 100644 index 000000000..3015073df --- /dev/null +++ b/examples/surf_collide/log.22Aug26.mpi_1.piston @@ -0,0 +1,123 @@ +SPARTA (24 Sep 2025) +Running on 1 MPI task(s) +################################################################################ +# a moving piston compressing a gas, with a vanishing outflow wall +# +# Covers surf_collide piston and surf_collide vanish, neither of which appears +# in any other deck. piston drives xlo inward so the gas is compressed and the +# temperature rises; vanish deletes anything reaching xhi, so Np falls. Both +# effects are visible in stats, which is what makes this a test rather than a +# smoke check. +# +# Note: +# - The "comm/sort" option to the "global" command is used to match MPI runs. +# - The "twopass" option is used to match Kokkos runs. +################################################################################ + +seed 12345 +dimension 3 +global gridcut 1.0e-5 comm/sort yes +boundary s s p +create_box 0 0.0001 0 0.0001 0 0.0001 +Created orthogonal box = (0 0 0) to (0.0001 0.0001 0.0001) +create_grid 10 10 10 +Created 1000 child grid cells + CPU time = 0.000950893 secs + create/ghost percent = 77.3777 22.6223 +balance_grid rcb part +Balance grid migrated 0 cells + CPU time = 0.00026696 secs + reassign/sort/migrate/ghost percent = 56.3897 1.69126 8.12182 33.7972 + +species air.species N2 N +mixture air N2 N vstream 0.0 0.0 0.0 temp 300.0 +mixture air N2 frac 1.0 +mixture air N frac 0.0 + +global nrho 7.07043E22 fnum 7.07043E6 + +collide vss air air.vss + +surf_collide pist piston 400.0 +surf_collide gone vanish +surf_collide wall specular + +bound_modify xlo collide pist +bound_modify xhi collide gone +bound_modify ylo collide wall +bound_modify yhi collide wall + +create_particles air n 20000 twopass +Created 20000 particles + CPU time = 0.00418998 secs + +compute t temp +stats 50 +stats_style step np nattempt ncoll c_t +timestep 1.0e-9 +run 300 +Memory usage per proc in Mbytes: + particles (ave,min,max) = 3.125 3.125 3.125 + grid (ave,min,max) = 1.51379 1.51379 1.51379 + surf (ave,min,max) = 0 0 0 + modify (ave,min,max) = 0 0 0 + total (ave,min,max) = 4.63879 4.63879 4.63879 +Step Np Natt Ncoll c_t + 0 20000 0 0 300.3378 + 50 17627 558 386 286.27355 + 100 15127 421 270 271.81282 + 150 12594 287 181 252.10156 + 200 10244 202 124 229.37031 + 250 8260 131 80 206.44468 + 300 6808 88 48 191.70048 +Loop time of 0.0765553 on 1 procs for 300 steps with 6808 particles +Performance: 3918.734 timesteps/s, 26.679 Mparticle-step/s + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Move | 0.037459 | 0.037459 | 0.037459 | 0.0 | 48.93 +Coll | 0.030355 | 0.030355 | 0.030355 | 0.0 | 39.65 +Sort | 0.0079577 | 0.0079577 | 0.0079577 | 0.0 | 10.39 +Comm | 0.00022831 | 0.00022831 | 0.00022831 | 0.0 | 0.30 +Modify | 0 | 0 | 0 | 0.0 | 0.00 +Output | 0.00040014 | 0.00040014 | 0.00040014 | 0.0 | 0.52 +MPI Sync| 0.00010937 | 0.00010937 | 0.00010937 | 0.0 | 0.14 +Other | | 4.625e-05 | | | 0.06 + +Particle moves = 3863687 (3.86M) +Cells touched = 4102976 (4.1M) +Particle comms = 0 (0K) +Boundary collides = 8618 (8.62K) +Boundary exits = 0 (0K) +SurfColl checks = 0 (0K) +SurfColl occurs = 0 (0K) +Surf reactions = 0 (0K) +Collide attempts = 97073 (97.1K) +Collide occurs = 63400 (63.4K) +Reactions = 0 (0K) +Particles stuck = 0 +Axisymm bad moves = 0 + +Particle-moves/CPUsec/proc: 5.04692e+07 +Particle-moves/step: 12879 +Cell-touches/particle/step: 1.06193 +Particle comm iterations/step: 1 +Particle fraction communicated: 0 +Particle fraction colliding with boundary: 0.00223051 +Particle fraction exiting boundary: 0 +Surface-checks/particle/step: 0 +Surface-collisions/particle/step: 0 +Surf-reactions/particle/step: 0 +Collision-attempts/particle/step: 0.0251244 +Collisions/particle/step: 0.0164092 +Reactions/particle/step: 0 + +Particles: 6808 ave 6808 max 6808 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Cells: 1000 ave 1000 max 1000 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +GhostCell: 0 ave 0 max 0 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +EmptyCell: 0 ave 0 max 0 min +Histogram: 1 0 0 0 0 0 0 0 0 0 diff --git a/examples/surf_collide/log.11Sep23.mpi_4.beam.adiabatic b/examples/surf_collide/log.22Aug26.mpi_4.beam.adiabatic similarity index 57% rename from examples/surf_collide/log.11Sep23.mpi_4.beam.adiabatic rename to examples/surf_collide/log.22Aug26.mpi_4.beam.adiabatic index a0a4888ef..f45300e81 100644 --- a/examples/surf_collide/log.11Sep23.mpi_4.beam.adiabatic +++ b/examples/surf_collide/log.22Aug26.mpi_4.beam.adiabatic @@ -1,4 +1,4 @@ -SPARTA (13 Apr 2023) +SPARTA (24 Sep 2025) Running on 4 MPI task(s) ################################################################################ # beam of particles striking the surface at an inclined angle - free molecular flow (no collisions) @@ -18,14 +18,14 @@ boundary oo oo so create_box -11 11 -11 11 0 10 Created orthogonal box = (-11 -11 0) to (11 11 10) create_grid 5 5 5 -WARNING: Could not acquire nearby ghost cells b/c grid partition is not clumped (/home/runner/work/sparta/sparta/src/grid.cpp:465) +WARNING: Could not acquire nearby ghost cells b/c grid partition is not clumped (/home/user/sparta/src/grid.cpp:486) Created 125 child grid cells - CPU time = 0.0030246 secs - create/ghost percent = 82.9961 17.0039 + CPU time = 0.0159224 secs + create/ghost percent = 30.3061 69.6939 balance_grid rcb cell Balance grid migrated 105 cells - CPU time = 0.0014433 secs - reassign/sort/migrate/ghost percent = 57.6596 0.879927 14.0858 27.3747 + CPU time = 0.000539744 secs + reassign/sort/migrate/ghost percent = 54.0855 0.433168 13.0627 32.4187 global nrho 1e10 fnum 1e6 @@ -38,8 +38,8 @@ mixture air O frac 0.2 surf_collide 1 adiabatic bound_modify zlo collide 1 -region circle cylinder z 0 -10 1 -INF INF -fix in emit/face/file air zhi data.beam beam_area nevery 100 region circle +region circle cylinder z 0 -10 1 INF INF +fix in emit/face/file air zhi data.beam beam_area nevery 100 region circle twopass #dump 2 image all 10 image.*.ppm type type pdiam 0.2 surf proc 0.01 size 512 512 zoom 1.75 gline no 0.005 #dump_modify 2 pad 4 @@ -53,37 +53,40 @@ Memory usage per proc in Mbytes: particles (ave,min,max) = 0 0 0 grid (ave,min,max) = 1.51379 1.51379 1.51379 surf (ave,min,max) = 0 0 0 + modify (ave,min,max) = 0 0 0 total (ave,min,max) = 1.51379 1.51379 1.51379 Step CPU Np Natt Ncoll Nscoll Nscheck 0 0 0 0 0 0 0 - 100 0.005699903 3118 0 0 0 0 - 200 0.022973611 6225 0 0 0 0 - 300 0.035604317 6949 0 0 0 0 - 400 0.048416623 6938 0 0 0 0 - 500 0.061910729 7006 0 0 0 0 - 600 0.075177735 6974 0 0 0 0 - 700 0.088207141 7006 0 0 0 0 - 800 0.10175825 6966 0 0 0 0 - 900 0.11516925 7058 0 0 0 0 - 1000 0.12915236 7069 0 0 0 0 -Loop time of 0.12922 on 4 procs for 1000 steps with 7069 particles + 100 0.002585304 3117 0 0 0 0 + 200 0.010478771 6221 0 0 0 0 + 300 0.018295555 6949 0 0 0 0 + 400 0.024854419 6942 0 0 0 0 + 500 0.031513761 7003 0 0 0 0 + 600 0.037438701 6988 0 0 0 0 + 700 0.043312494 6995 0 0 0 0 + 800 0.049744355 6960 0 0 0 0 + 900 0.056101488 7057 0 0 0 0 + 1000 0.063190353 7078 0 0 0 0 +Loop time of 0.0632396 on 4 procs for 1000 steps with 7078 particles +Performance: 15812.865 timesteps/s, 111.923 Mparticle-step/s MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Move | 0.0056796 | 0.014615 | 0.02266 | 5.1 | 11.31 +Move | 0.0053029 | 0.013338 | 0.02041 | 4.7 | 21.09 Coll | 0 | 0 | 0 | 0.0 | 0.00 Sort | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.055441 | 0.067806 | 0.082888 | 4.7 | 52.47 -Modify | 7.41e-05 | 0.0019413 | 0.0075235 | 7.3 | 1.50 -Output | 0.0003199 | 0.0004802 | 0.0007344 | 0.0 | 0.37 -Other | | 0.04438 | | | 34.34 - -Particle moves = 5075752 (5.08M) -Cells touched = 5405012 (5.41M) -Particle comms = 193584 (0.194M) -Boundary collides = 28030 (28K) -Boundary exits = 24101 (24.1K) +Comm | 0.018715 | 0.020176 | 0.02118 | 0.6 | 31.90 +Modify | 8.2906e-05 | 0.0019286 | 0.0074474 | 7.3 | 3.05 +Output | 0.00065674 | 0.0007746 | 0.0011268 | 0.0 | 1.22 +MPI Sync| 0.020834 | 0.026955 | 0.038413 | 4.3 | 42.62 +Other | | 6.679e-05 | | | 0.11 + +Particle moves = 5074673 (5.07M) +Cells touched = 5403354 (5.4M) +Particle comms = 193300 (0.193M) +Boundary collides = 28019 (28K) +Boundary exits = 24076 (24.1K) SurfColl checks = 0 (0K) SurfColl occurs = 0 (0K) Surf reactions = 0 (0K) @@ -93,13 +96,13 @@ Reactions = 0 (0K) Particles stuck = 0 Axisymm bad moves = 0 -Particle-moves/CPUsec/proc: 9.82001e+06 -Particle-moves/step: 5075.75 -Cell-touches/particle/step: 1.06487 -Particle comm iterations/step: 1.756 -Particle fraction communicated: 0.038139 -Particle fraction colliding with boundary: 0.00552233 -Particle fraction exiting boundary: 0.00474826 +Particle-moves/CPUsec/proc: 2.00613e+07 +Particle-moves/step: 5074.67 +Cell-touches/particle/step: 1.06477 +Particle comm iterations/step: 1.755 +Particle fraction communicated: 0.0380911 +Particle fraction colliding with boundary: 0.00552134 +Particle fraction exiting boundary: 0.00474435 Surface-checks/particle/step: 0 Surface-collisions/particle/step: 0 Surf-reactions/particle/step: 0 @@ -107,7 +110,7 @@ Collision-attempts/particle/step: 0 Collisions/particle/step: 0 Reactions/particle/step: 0 -Particles: 1767.25 ave 3379 max 177 min +Particles: 1769.5 ave 3372 max 177 min Histogram: 2 0 0 0 0 0 0 0 0 2 Cells: 31.25 ave 32 max 31 min Histogram: 3 0 0 0 0 0 0 0 0 1 diff --git a/examples/surf_collide/log.11Sep23.mpi_4.beam.cll b/examples/surf_collide/log.22Aug26.mpi_4.beam.cll similarity index 57% rename from examples/surf_collide/log.11Sep23.mpi_4.beam.cll rename to examples/surf_collide/log.22Aug26.mpi_4.beam.cll index 857bdf862..5b948fcd7 100644 --- a/examples/surf_collide/log.11Sep23.mpi_4.beam.cll +++ b/examples/surf_collide/log.22Aug26.mpi_4.beam.cll @@ -1,4 +1,4 @@ -SPARTA (13 Apr 2023) +SPARTA (24 Sep 2025) Running on 4 MPI task(s) ################################################################################ # beam of particles striking the surface at an inclined angle - free molecular flow (no collisions) @@ -18,14 +18,14 @@ boundary oo oo so create_box -11 11 -11 11 0 10 Created orthogonal box = (-11 -11 0) to (11 11 10) create_grid 5 5 5 -WARNING: Could not acquire nearby ghost cells b/c grid partition is not clumped (/home/runner/work/sparta/sparta/src/grid.cpp:465) +WARNING: Could not acquire nearby ghost cells b/c grid partition is not clumped (/home/user/sparta/src/grid.cpp:486) Created 125 child grid cells - CPU time = 0.0018753 secs - create/ghost percent = 91.916 8.08404 + CPU time = 0.00116668 secs + create/ghost percent = 96.1501 3.84991 balance_grid rcb cell Balance grid migrated 105 cells - CPU time = 0.0008151 secs - reassign/sort/migrate/ghost percent = 57.7598 0.711569 13.9369 27.5917 + CPU time = 0.000444854 secs + reassign/sort/migrate/ghost percent = 52.8045 0.448012 13.0425 33.705 global nrho 1e10 fnum 1e6 @@ -38,8 +38,8 @@ mixture air O frac 0.2 surf_collide 1 cll 300.0 0.8 0.8 0.8 0.8 #partial 0.5 bound_modify zlo collide 1 -region circle cylinder z 0 -10 1 -INF INF -fix in emit/face/file air zhi data.beam beam_area nevery 100 region circle +region circle cylinder z 0 -10 1 INF INF +fix in emit/face/file air zhi data.beam beam_area nevery 100 region circle twopass #dump 2 image all 10 image.*.ppm type type pdiam 0.2 surf proc 0.01 size 512 512 zoom 1.75 gline no 0.005 #dump_modify 2 pad 4 @@ -53,37 +53,40 @@ Memory usage per proc in Mbytes: particles (ave,min,max) = 0 0 0 grid (ave,min,max) = 1.51379 1.51379 1.51379 surf (ave,min,max) = 0 0 0 + modify (ave,min,max) = 0 0 0 total (ave,min,max) = 1.51379 1.51379 1.51379 Step CPU Np Natt Ncoll Nscoll Nscheck 0 0 0 0 0 0 0 - 100 0.003726401 3118 0 0 0 0 - 200 0.014044306 6225 0 0 0 0 - 300 0.025720311 8796 0 0 0 0 - 400 0.038098717 9425 0 0 0 0 - 500 0.051494123 9581 0 0 0 0 - 600 0.064769229 9643 0 0 0 0 - 700 0.078656336 9703 0 0 0 0 - 800 0.091820042 9747 0 0 0 0 - 900 0.10472165 9786 0 0 0 0 - 1000 0.11814605 9768 0 0 0 0 -Loop time of 0.118211 on 4 procs for 1000 steps with 9768 particles + 100 0.00216701 3117 0 0 0 0 + 200 0.009745305 6221 0 0 0 0 + 300 0.017215288 8789 0 0 0 0 + 400 0.023365002 9409 0 0 0 0 + 500 0.029969977 9599 0 0 0 0 + 600 0.036303256 9621 0 0 0 0 + 700 0.042055305 9693 0 0 0 0 + 800 0.048615111 9753 0 0 0 0 + 900 0.054833898 9789 0 0 0 0 + 1000 0.061843203 9760 0 0 0 0 +Loop time of 0.0619071 on 4 procs for 1000 steps with 9760 particles +Performance: 16153.226 timesteps/s, 157.655 Mparticle-step/s MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Move | 0.011638 | 0.018883 | 0.029 | 5.1 | 15.97 +Move | 0.010717 | 0.016635 | 0.025757 | 4.7 | 26.87 Coll | 0 | 0 | 0 | 0.0 | 0.00 Sort | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.052078 | 0.06083 | 0.071587 | 3.4 | 51.46 -Modify | 6.99e-05 | 0.0019592 | 0.0076108 | 7.4 | 1.66 -Output | 0.0002841 | 0.00050913 | 0.000852 | 0.0 | 0.43 -Other | | 0.03603 | | | 30.48 - -Particle moves = 6530753 (6.53M) -Cells touched = 6857152 (6.86M) -Particle comms = 184478 (0.184M) -Boundary collides = 28030 (28K) -Boundary exits = 21402 (21.4K) +Comm | 0.018256 | 0.019047 | 0.019745 | 0.4 | 30.77 +Modify | 7.9725e-05 | 0.001923 | 0.0074108 | 7.2 | 3.11 +Output | 0.00010648 | 0.00021555 | 0.00053908 | 0.0 | 0.35 +MPI Sync| 0.01659 | 0.024021 | 0.0317 | 3.5 | 38.80 +Other | | 6.532e-05 | | | 0.11 + +Particle moves = 6528079 (6.53M) +Cells touched = 6854251 (6.85M) +Particle comms = 184436 (0.184M) +Boundary collides = 28019 (28K) +Boundary exits = 21394 (21.4K) SurfColl checks = 0 (0K) SurfColl occurs = 0 (0K) Surf reactions = 0 (0K) @@ -93,13 +96,13 @@ Reactions = 0 (0K) Particles stuck = 0 Axisymm bad moves = 0 -Particle-moves/CPUsec/proc: 1.38116e+07 -Particle-moves/step: 6530.75 -Cell-touches/particle/step: 1.04998 -Particle comm iterations/step: 1.823 -Particle fraction communicated: 0.0282476 -Particle fraction colliding with boundary: 0.004292 -Particle fraction exiting boundary: 0.00327711 +Particle-moves/CPUsec/proc: 2.63624e+07 +Particle-moves/step: 6528.08 +Cell-touches/particle/step: 1.04996 +Particle comm iterations/step: 1.816 +Particle fraction communicated: 0.0282527 +Particle fraction colliding with boundary: 0.00429207 +Particle fraction exiting boundary: 0.00327723 Surface-checks/particle/step: 0 Surface-collisions/particle/step: 0 Surf-reactions/particle/step: 0 @@ -107,7 +110,7 @@ Collision-attempts/particle/step: 0 Collisions/particle/step: 0 Reactions/particle/step: 0 -Particles: 2442 ave 4565 max 365 min +Particles: 2440 ave 4559 max 372 min Histogram: 1 0 1 0 0 0 0 1 0 1 Cells: 31.25 ave 32 max 31 min Histogram: 3 0 0 0 0 0 0 0 0 1 diff --git a/examples/surf_collide/log.11Sep23.mpi_4.beam.diffuse b/examples/surf_collide/log.22Aug26.mpi_4.beam.diffuse similarity index 57% rename from examples/surf_collide/log.11Sep23.mpi_4.beam.diffuse rename to examples/surf_collide/log.22Aug26.mpi_4.beam.diffuse index 93d549a59..c774b82fe 100644 --- a/examples/surf_collide/log.11Sep23.mpi_4.beam.diffuse +++ b/examples/surf_collide/log.22Aug26.mpi_4.beam.diffuse @@ -1,4 +1,4 @@ -SPARTA (13 Apr 2023) +SPARTA (24 Sep 2025) Running on 4 MPI task(s) ################################################################################ # beam of particles striking the surface at an inclined angle - free molecular flow (no collisions) @@ -18,14 +18,14 @@ boundary oo oo so create_box -11 11 -11 11 0 10 Created orthogonal box = (-11 -11 0) to (11 11 10) create_grid 5 5 5 -WARNING: Could not acquire nearby ghost cells b/c grid partition is not clumped (/home/runner/work/sparta/sparta/src/grid.cpp:465) +WARNING: Could not acquire nearby ghost cells b/c grid partition is not clumped (/home/user/sparta/src/grid.cpp:486) Created 125 child grid cells - CPU time = 0.0018056 secs - create/ghost percent = 91.7977 8.20226 + CPU time = 0.00152922 secs + create/ghost percent = 95.2209 4.77911 balance_grid rcb cell Balance grid migrated 105 cells - CPU time = 0.0009312 secs - reassign/sort/migrate/ghost percent = 52.8458 0.579897 12.3389 34.2354 + CPU time = 0.000535424 secs + reassign/sort/migrate/ghost percent = 54.8087 0.774153 15.1982 29.2189 global nrho 1e10 fnum 1e6 @@ -38,8 +38,8 @@ mixture air O frac 0.2 surf_collide 1 diffuse 300 0.5 bound_modify zlo collide 1 -region circle cylinder z 0 -10 1 -INF INF -fix in emit/face/file air zhi data.beam beam_area nevery 100 region circle +region circle cylinder z 0 -10 1 INF INF +fix in emit/face/file air zhi data.beam beam_area nevery 100 region circle twopass #dump 2 image all 10 image.*.ppm type type pdiam 0.2 surf proc 0.01 size 512 512 zoom 1.75 gline no 0.005 #dump_modify 2 pad 4 @@ -53,37 +53,40 @@ Memory usage per proc in Mbytes: particles (ave,min,max) = 0 0 0 grid (ave,min,max) = 1.51379 1.51379 1.51379 surf (ave,min,max) = 0 0 0 + modify (ave,min,max) = 0 0 0 total (ave,min,max) = 1.51379 1.51379 1.51379 Step CPU Np Natt Ncoll Nscoll Nscheck 0 0 0 0 0 0 0 - 100 0.004592503 3118 0 0 0 0 - 200 0.015742808 6225 0 0 0 0 - 300 0.028496614 7656 0 0 0 0 - 400 0.04160932 8146 0 0 0 0 - 500 0.055141226 8392 0 0 0 0 - 600 0.069245732 8416 0 0 0 0 - 700 0.082388238 8454 0 0 0 0 - 800 0.095699944 8499 0 0 0 0 - 900 0.10931855 8506 0 0 0 0 - 1000 0.12309506 8558 0 0 0 0 -Loop time of 0.123197 on 4 procs for 1000 steps with 8558 particles + 100 0.002848793 3117 0 0 0 0 + 200 0.012590063 6221 0 0 0 0 + 300 0.021336271 7653 0 0 0 0 + 400 0.02872164 8148 0 0 0 0 + 500 0.034959705 8368 0 0 0 0 + 600 0.041203018 8425 0 0 0 0 + 700 0.046897815 8446 0 0 0 0 + 800 0.054011041 8501 0 0 0 0 + 900 0.061894573 8506 0 0 0 0 + 1000 0.069242572 8558 0 0 0 0 +Loop time of 0.0693378 on 4 procs for 1000 steps with 8558 particles +Performance: 14422.138 timesteps/s, 123.425 Mparticle-step/s MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Move | 0.011765 | 0.017025 | 0.022189 | 3.7 | 13.82 +Move | 0.010788 | 0.016857 | 0.023917 | 4.1 | 24.31 Coll | 0 | 0 | 0 | 0.0 | 0.00 Sort | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.049258 | 0.060876 | 0.073865 | 3.9 | 49.41 -Modify | 7.62e-05 | 0.0021171 | 0.0082222 | 7.7 | 1.72 -Output | 0.0004087 | 0.0006322 | 0.0011103 | 0.0 | 0.51 -Other | | 0.04255 | | | 34.54 - -Particle moves = 6216741 (6.22M) -Cells touched = 6548784 (6.55M) -Particle comms = 196264 (0.196M) -Boundary collides = 28030 (28K) -Boundary exits = 22612 (22.6K) +Comm | 0.021662 | 0.022329 | 0.023269 | 0.5 | 32.20 +Modify | 9.4303e-05 | 0.0020525 | 0.0078671 | 7.4 | 2.96 +Output | 0.0001292 | 0.00027055 | 0.00069223 | 0.0 | 0.39 +MPI Sync| 0.021826 | 0.027755 | 0.036555 | 3.2 | 40.03 +Other | | 7.401e-05 | | | 0.11 + +Particle moves = 6214484 (6.21M) +Cells touched = 6546435 (6.55M) +Particle comms = 196238 (0.196M) +Boundary collides = 28019 (28K) +Boundary exits = 22596 (22.6K) SurfColl checks = 0 (0K) SurfColl occurs = 0 (0K) Surf reactions = 0 (0K) @@ -93,13 +96,13 @@ Reactions = 0 (0K) Particles stuck = 0 Axisymm bad moves = 0 -Particle-moves/CPUsec/proc: 1.26154e+07 -Particle-moves/step: 6216.74 -Cell-touches/particle/step: 1.05341 -Particle comm iterations/step: 1.801 -Particle fraction communicated: 0.0315702 -Particle fraction colliding with boundary: 0.00450879 -Particle fraction exiting boundary: 0.00363728 +Particle-moves/CPUsec/proc: 2.24065e+07 +Particle-moves/step: 6214.48 +Cell-touches/particle/step: 1.05342 +Particle comm iterations/step: 1.806 +Particle fraction communicated: 0.0315775 +Particle fraction colliding with boundary: 0.00450866 +Particle fraction exiting boundary: 0.00363602 Surface-checks/particle/step: 0 Surface-collisions/particle/step: 0 Surf-reactions/particle/step: 0 @@ -107,7 +110,7 @@ Collision-attempts/particle/step: 0 Collisions/particle/step: 0 Reactions/particle/step: 0 -Particles: 2139.5 ave 3703 max 605 min +Particles: 2139.5 ave 3694 max 611 min Histogram: 2 0 0 0 0 0 0 0 0 2 Cells: 31.25 ave 32 max 31 min Histogram: 3 0 0 0 0 0 0 0 0 1 diff --git a/examples/surf_collide/log.11Sep23.mpi_4.beam.impulsive b/examples/surf_collide/log.22Aug26.mpi_4.beam.impulsive similarity index 61% rename from examples/surf_collide/log.11Sep23.mpi_4.beam.impulsive rename to examples/surf_collide/log.22Aug26.mpi_4.beam.impulsive index d0a60a9c8..c2bad7239 100644 --- a/examples/surf_collide/log.11Sep23.mpi_4.beam.impulsive +++ b/examples/surf_collide/log.22Aug26.mpi_4.beam.impulsive @@ -1,4 +1,4 @@ -SPARTA (13 Apr 2023) +SPARTA (24 Sep 2025) Running on 4 MPI task(s) ################################################################################ # beam of particles striking the surface at an inclined angle - free molecular flow (no collisions) @@ -18,14 +18,14 @@ boundary oo oo so create_box -11 11 -11 11 0 10 Created orthogonal box = (-11 -11 0) to (11 11 10) create_grid 1 1 1 -WARNING: Could not acquire nearby ghost cells b/c grid partition is not clumped (/home/runner/work/sparta/sparta/src/grid.cpp:465) +WARNING: Could not acquire nearby ghost cells b/c grid partition is not clumped (/home/user/sparta/src/grid.cpp:486) Created 1 child grid cells - CPU time = 0.000965101 secs - create/ghost percent = 85.5145 14.4855 + CPU time = 0.00142621 secs + create/ghost percent = 93.1012 6.8988 balance_grid rcb cell Balance grid migrated 0 cells - CPU time = 0.0005519 secs - reassign/sort/migrate/ghost percent = 76.8074 0.9422 6.83095 15.4195 + CPU time = 0.000287285 secs + reassign/sort/migrate/ghost percent = 81.6409 0.641871 5.58052 12.1367 global nrho 1e10 fnum 1e6 @@ -40,8 +40,8 @@ surf_collide 1 impulsive 1000.0 softsphere 0.2 50 2000 60 5 75 #double 10 bound_modify zlo collide 1 -region circle cylinder z 0 -10 1 -INF INF -fix in emit/face/file air zhi data.beam beam_area nevery 100 region circle +region circle cylinder z 0 -10 1 INF INF +fix in emit/face/file air zhi data.beam beam_area nevery 100 region circle twopass #dump 2 image all 10 image.*.ppm type type pdiam 0.2 surf proc 0.01 size 512 512 zoom 1.75 gline no 0.005 #dump_modify 2 pad 4 @@ -55,37 +55,40 @@ Memory usage per proc in Mbytes: particles (ave,min,max) = 0 0 0 grid (ave,min,max) = 0.435669 0.0762939 1.51379 surf (ave,min,max) = 0 0 0 + modify (ave,min,max) = 0 0 0 total (ave,min,max) = 0.435669 0.0762939 1.51379 Step CPU Np Natt Ncoll Nscoll Nscheck 0 0 0 0 0 0 0 - 100 0.003060102 3118 0 0 0 0 - 200 0.019896609 6225 0 0 0 0 - 300 0.041432719 7260 0 0 0 0 - 400 0.063845229 7486 0 0 0 0 - 500 0.08667784 7595 0 0 0 0 - 600 0.10963615 7584 0 0 0 0 - 700 0.13230676 7547 0 0 0 0 - 800 0.15511167 7578 0 0 0 0 - 900 0.17812408 7668 0 0 0 0 - 1000 0.20155919 7656 0 0 0 0 -Loop time of 0.201662 on 4 procs for 1000 steps with 7656 particles + 100 0.002671174 3117 0 0 0 0 + 200 0.011946623 6221 0 0 0 0 + 300 0.025990675 7246 0 0 0 0 + 400 0.039727268 7492 0 0 0 0 + 500 0.051333855 7560 0 0 0 0 + 600 0.063719947 7559 0 0 0 0 + 700 0.077051043 7546 0 0 0 0 + 800 0.090165994 7609 0 0 0 0 + 900 0.099924606 7697 0 0 0 0 + 1000 0.1097949 7639 0 0 0 0 +Loop time of 0.109866 on 4 procs for 1000 steps with 7639 particles +Performance: 9101.964 timesteps/s, 69.530 Mparticle-step/s MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Move | 0.0001156 | 0.023354 | 0.093054 | 26.3 | 11.58 +Move | 9.3898e-05 | 0.024115 | 0.096175 | 26.8 | 21.95 Coll | 0 | 0 | 0 | 0.0 | 0.00 Sort | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.006473 | 0.075989 | 0.19338 | 28.0 | 37.68 -Modify | 6.15e-05 | 0.0019357 | 0.0075437 | 7.4 | 0.96 -Output | 0.0002986 | 0.00050982 | 0.0006668 | 0.0 | 0.25 -Other | | 0.09987 | | | 49.53 - -Particle moves = 5231166 (5.23M) -Cells touched = 5231166 (5.23M) +Comm | 0.0016834 | 0.0019408 | 0.0021829 | 0.4 | 1.77 +Modify | 7.2878e-05 | 0.0023449 | 0.0091514 | 8.1 | 2.13 +Output | 0.00021272 | 0.00037356 | 0.00084774 | 0.0 | 0.34 +MPI Sync| 0.0017648 | 0.081027 | 0.10773 | 16.1 | 73.75 +Other | | 6.497e-05 | | | 0.06 + +Particle moves = 5231445 (5.23M) +Cells touched = 5231445 (5.23M) Particle comms = 0 (0K) -Boundary collides = 28030 (28K) -Boundary exits = 23514 (23.5K) +Boundary collides = 28019 (28K) +Boundary exits = 23515 (23.5K) SurfColl checks = 0 (0K) SurfColl occurs = 0 (0K) Surf reactions = 0 (0K) @@ -95,13 +98,13 @@ Reactions = 0 (0K) Particles stuck = 0 Axisymm bad moves = 0 -Particle-moves/CPUsec/proc: 6.48507e+06 -Particle-moves/step: 5231.17 +Particle-moves/CPUsec/proc: 1.19041e+07 +Particle-moves/step: 5231.44 Cell-touches/particle/step: 1 Particle comm iterations/step: 1 Particle fraction communicated: 0 -Particle fraction colliding with boundary: 0.00535827 -Particle fraction exiting boundary: 0.00449498 +Particle fraction colliding with boundary: 0.00535588 +Particle fraction exiting boundary: 0.00449493 Surface-checks/particle/step: 0 Surface-collisions/particle/step: 0 Surf-reactions/particle/step: 0 @@ -109,7 +112,7 @@ Collision-attempts/particle/step: 0 Collisions/particle/step: 0 Reactions/particle/step: 0 -Particles: 1914 ave 7656 max 0 min +Particles: 1909.75 ave 7639 max 0 min Histogram: 3 0 0 0 0 0 0 0 0 1 Cells: 0.25 ave 1 max 0 min Histogram: 3 0 0 0 0 0 0 0 0 1 diff --git a/examples/surf_collide/log.11Sep23.mpi_4.beam.specular b/examples/surf_collide/log.22Aug26.mpi_4.beam.specular similarity index 59% rename from examples/surf_collide/log.11Sep23.mpi_4.beam.specular rename to examples/surf_collide/log.22Aug26.mpi_4.beam.specular index bd932465f..f6f050fd8 100644 --- a/examples/surf_collide/log.11Sep23.mpi_4.beam.specular +++ b/examples/surf_collide/log.22Aug26.mpi_4.beam.specular @@ -1,4 +1,4 @@ -SPARTA (13 Apr 2023) +SPARTA (24 Sep 2025) Running on 4 MPI task(s) ################################################################################ # beam of particles striking the surface at an inclined angle - free molecular flow (no collisions) @@ -18,14 +18,14 @@ boundary oo oo so create_box -11 11 -11 11 0 10 Created orthogonal box = (-11 -11 0) to (11 11 10) create_grid 5 5 5 -WARNING: Could not acquire nearby ghost cells b/c grid partition is not clumped (/home/runner/work/sparta/sparta/src/grid.cpp:465) +WARNING: Could not acquire nearby ghost cells b/c grid partition is not clumped (/home/user/sparta/src/grid.cpp:486) Created 125 child grid cells - CPU time = 0.0018578 secs - create/ghost percent = 92.518 7.48197 + CPU time = 0.00125364 secs + create/ghost percent = 94.7495 5.25048 balance_grid rcb cell Balance grid migrated 105 cells - CPU time = 0.000849101 secs - reassign/sort/migrate/ghost percent = 58.3913 0.683075 13.7322 27.1935 + CPU time = 0.000544402 secs + reassign/sort/migrate/ghost percent = 58.0352 0.336149 11.6431 29.9856 global nrho 1e10 fnum 1e6 @@ -38,8 +38,8 @@ mixture air O frac 0.2 surf_collide 1 specular bound_modify zlo collide 1 -region circle cylinder z 0 -10 1 -INF INF -fix in emit/face/file air zhi data.beam beam_area nevery 100 region circle +region circle cylinder z 0 -10 1 INF INF +fix in emit/face/file air zhi data.beam beam_area nevery 100 region circle twopass #dump 2 image all 10 image.*.ppm type type pdiam 0.2 surf proc 0.01 size 512 512 zoom 1.75 gline no 0.005 #dump_modify 2 pad 4 @@ -53,37 +53,40 @@ Memory usage per proc in Mbytes: particles (ave,min,max) = 0 0 0 grid (ave,min,max) = 1.51379 1.51379 1.51379 surf (ave,min,max) = 0 0 0 + modify (ave,min,max) = 0 0 0 total (ave,min,max) = 1.51379 1.51379 1.51379 Step CPU Np Natt Ncoll Nscoll Nscheck 0 0 0 0 0 0 0 - 100 0.003532102 3101 0 0 0 0 - 200 0.015454207 6270 0 0 0 0 - 300 0.026520812 6382 0 0 0 0 - 400 0.035287016 6354 0 0 0 0 - 500 0.04361492 6338 0 0 0 0 - 600 0.051725924 6334 0 0 0 0 - 700 0.059908728 6389 0 0 0 0 - 800 0.068623732 6412 0 0 0 0 - 900 0.077407836 6359 0 0 0 0 - 1000 0.085743939 6335 0 0 0 0 -Loop time of 0.0858001 on 4 procs for 1000 steps with 6335 particles + 100 0.002233132 3098 0 0 0 0 + 200 0.011286359 6270 0 0 0 0 + 300 0.018753148 6383 0 0 0 0 + 400 0.024503886 6349 0 0 0 0 + 500 0.029879855 6335 0 0 0 0 + 600 0.035372498 6334 0 0 0 0 + 700 0.040671872 6393 0 0 0 0 + 800 0.046359206 6414 0 0 0 0 + 900 0.051769251 6355 0 0 0 0 + 1000 0.05658519 6334 0 0 0 0 +Loop time of 0.0566361 on 4 procs for 1000 steps with 6334 particles +Performance: 17656.590 timesteps/s, 111.837 Mparticle-step/s MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Move | 0.0083187 | 0.01376 | 0.016764 | 2.8 | 16.04 +Move | 0.0073784 | 0.012771 | 0.016996 | 3.1 | 22.55 Coll | 0 | 0 | 0 | 0.0 | 0.00 Sort | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.033268 | 0.040476 | 0.049996 | 3.0 | 47.17 -Modify | 7.29e-05 | 0.0019585 | 0.0076022 | 7.4 | 2.28 -Output | 0.0002072 | 0.00045235 | 0.0008587 | 0.0 | 0.53 -Other | | 0.02915 | | | 33.98 - -Particle moves = 5383479 (5.38M) -Cells touched = 5704605 (5.7M) -Particle comms = 204034 (0.204M) -Boundary collides = 28312 (28.3K) -Boundary exits = 25108 (25.1K) +Comm | 0.017105 | 0.017769 | 0.018269 | 0.3 | 31.37 +Modify | 8.3008e-05 | 0.0018736 | 0.007235 | 7.2 | 3.31 +Output | 0.00013618 | 0.0002755 | 0.00069175 | 0.0 | 0.49 +MPI Sync| 0.021476 | 0.023887 | 0.027043 | 1.3 | 42.18 +Other | | 5.996e-05 | | | 0.11 + +Particle moves = 5382677 (5.38M) +Cells touched = 5703755 (5.7M) +Particle comms = 204004 (0.204M) +Boundary collides = 28306 (28.3K) +Boundary exits = 25105 (25.1K) SurfColl checks = 0 (0K) SurfColl occurs = 0 (0K) Surf reactions = 0 (0K) @@ -93,13 +96,13 @@ Reactions = 0 (0K) Particles stuck = 0 Axisymm bad moves = 0 -Particle-moves/CPUsec/proc: 1.56861e+07 -Particle-moves/step: 5383.48 +Particle-moves/CPUsec/proc: 2.37599e+07 +Particle-moves/step: 5382.68 Cell-touches/particle/step: 1.05965 Particle comm iterations/step: 1.104 -Particle fraction communicated: 0.0379 -Particle fraction colliding with boundary: 0.00525905 -Particle fraction exiting boundary: 0.0046639 +Particle fraction communicated: 0.0379001 +Particle fraction colliding with boundary: 0.00525872 +Particle fraction exiting boundary: 0.00466404 Surface-checks/particle/step: 0 Surface-collisions/particle/step: 0 Surf-reactions/particle/step: 0 @@ -107,7 +110,7 @@ Collision-attempts/particle/step: 0 Collisions/particle/step: 0 Reactions/particle/step: 0 -Particles: 1583.75 ave 3145 max 0 min +Particles: 1583.5 ave 3141 max 0 min Histogram: 2 0 0 0 0 0 0 0 0 2 Cells: 31.25 ave 32 max 31 min Histogram: 3 0 0 0 0 0 0 0 0 1 diff --git a/examples/surf_collide/log.11Sep23.mpi_4.beam.td b/examples/surf_collide/log.22Aug26.mpi_4.beam.td similarity index 61% rename from examples/surf_collide/log.11Sep23.mpi_4.beam.td rename to examples/surf_collide/log.22Aug26.mpi_4.beam.td index f480aae93..9cfef5a44 100644 --- a/examples/surf_collide/log.11Sep23.mpi_4.beam.td +++ b/examples/surf_collide/log.22Aug26.mpi_4.beam.td @@ -1,4 +1,4 @@ -SPARTA (13 Apr 2023) +SPARTA (24 Sep 2025) Running on 4 MPI task(s) ################################################################################ # beam of particles striking the surface at an inclined angle - free molecular flow (no collisions) @@ -18,14 +18,14 @@ boundary oo oo so create_box -11 11 -11 11 0 10 Created orthogonal box = (-11 -11 0) to (11 11 10) create_grid 1 1 1 -WARNING: Could not acquire nearby ghost cells b/c grid partition is not clumped (/home/runner/work/sparta/sparta/src/grid.cpp:465) +WARNING: Could not acquire nearby ghost cells b/c grid partition is not clumped (/home/user/sparta/src/grid.cpp:486) Created 1 child grid cells - CPU time = 0.0025614 secs - create/ghost percent = 92.5471 7.45295 + CPU time = 0.00118115 secs + create/ghost percent = 90.2785 9.72153 balance_grid rcb cell Balance grid migrated 0 cells - CPU time = 0.0009947 secs - reassign/sort/migrate/ghost percent = 72.273 1.53815 7.26852 18.9203 + CPU time = 0.00024771 secs + reassign/sort/migrate/ghost percent = 81.9155 0.597877 6.3098 11.1768 global nrho 1e10 fnum 1e6 @@ -42,8 +42,8 @@ surf_collide 1 td 1000.0 #barrier 1000 bound_modify zlo collide 1 -region circle cylinder z 0 -10 1 -INF INF -fix in emit/face/file air zhi data.beam beam_area nevery 100 region circle +region circle cylinder z 0 -10 1 INF INF +fix in emit/face/file air zhi data.beam beam_area nevery 100 region circle twopass #dump 2 image all 10 image.*.ppm type type pdiam 0.2 surf proc 0.01 size 512 512 zoom 1.75 gline no 0.005 #dump_modify 2 pad 4 @@ -57,37 +57,40 @@ Memory usage per proc in Mbytes: particles (ave,min,max) = 0 0 0 grid (ave,min,max) = 0.435669 0.0762939 1.51379 surf (ave,min,max) = 0 0 0 + modify (ave,min,max) = 0 0 0 total (ave,min,max) = 0.435669 0.0762939 1.51379 Step CPU Np Natt Ncoll Nscoll Nscheck 0 0 0 0 0 0 0 - 100 0.007029503 3118 0 0 0 0 - 200 0.020670909 6225 0 0 0 0 - 300 0.041914619 7461 0 0 0 0 - 400 0.064784029 7628 0 0 0 0 - 500 0.08754384 7682 0 0 0 0 - 600 0.10386265 7701 0 0 0 0 - 700 0.11907315 7763 0 0 0 0 - 800 0.13399006 7772 0 0 0 0 - 900 0.14890517 7847 0 0 0 0 - 1000 0.16390548 7765 0 0 0 0 -Loop time of 0.16399 on 4 procs for 1000 steps with 7765 particles + 100 0.002143635 3117 0 0 0 0 + 200 0.005881792 6221 0 0 0 0 + 300 0.011647181 7450 0 0 0 0 + 400 0.01797286 7621 0 0 0 0 + 500 0.0276373 7669 0 0 0 0 + 600 0.039105989 7699 0 0 0 0 + 700 0.045693003 7768 0 0 0 0 + 800 0.052235927 7777 0 0 0 0 + 900 0.058604911 7841 0 0 0 0 + 1000 0.065068459 7783 0 0 0 0 +Loop time of 0.0651252 on 4 procs for 1000 steps with 7783 particles +Performance: 15355.029 timesteps/s, 119.508 Mparticle-step/s MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Move | 0.000119 | 0.01346 | 0.053473 | 19.9 | 8.21 +Move | 8.8932e-05 | 0.013668 | 0.054399 | 20.1 | 20.99 Coll | 0 | 0 | 0 | 0.0 | 0.00 Sort | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.0030248 | 0.080635 | 0.12657 | 17.7 | 49.17 -Modify | 6.47e-05 | 0.001936 | 0.0075023 | 7.3 | 1.18 -Output | 0.0005545 | 0.00067405 | 0.0008876 | 0.0 | 0.41 -Other | | 0.06728 | | | 41.03 - -Particle moves = 5391255 (5.39M) -Cells touched = 5391255 (5.39M) +Comm | 0.0012703 | 0.0015177 | 0.0016712 | 0.4 | 2.33 +Modify | 6.6734e-05 | 0.00189 | 0.0073543 | 7.3 | 2.90 +Output | 0.00010114 | 0.00020536 | 0.00051472 | 0.0 | 0.32 +MPI Sync| 0.0011553 | 0.047787 | 0.06354 | 12.3 | 73.38 +Other | | 5.71e-05 | | | 0.09 + +Particle moves = 5389397 (5.39M) +Cells touched = 5389397 (5.39M) Particle comms = 0 (0K) -Boundary collides = 28030 (28K) -Boundary exits = 23405 (23.4K) +Boundary collides = 28019 (28K) +Boundary exits = 23371 (23.4K) SurfColl checks = 0 (0K) SurfColl occurs = 0 (0K) Surf reactions = 0 (0K) @@ -97,13 +100,13 @@ Reactions = 0 (0K) Particles stuck = 0 Axisymm bad moves = 0 -Particle-moves/CPUsec/proc: 8.21888e+06 -Particle-moves/step: 5391.26 +Particle-moves/CPUsec/proc: 2.06886e+07 +Particle-moves/step: 5389.4 Cell-touches/particle/step: 1 Particle comm iterations/step: 1 Particle fraction communicated: 0 -Particle fraction colliding with boundary: 0.00519916 -Particle fraction exiting boundary: 0.00434129 +Particle fraction colliding with boundary: 0.00519891 +Particle fraction exiting boundary: 0.00433648 Surface-checks/particle/step: 0 Surface-collisions/particle/step: 0 Surf-reactions/particle/step: 0 @@ -111,7 +114,7 @@ Collision-attempts/particle/step: 0 Collisions/particle/step: 0 Reactions/particle/step: 0 -Particles: 1941.25 ave 7765 max 0 min +Particles: 1945.75 ave 7783 max 0 min Histogram: 3 0 0 0 0 0 0 0 0 1 Cells: 0.25 ave 1 max 0 min Histogram: 3 0 0 0 0 0 0 0 0 1 diff --git a/examples/surf_collide/log.22Aug26.mpi_4.piston b/examples/surf_collide/log.22Aug26.mpi_4.piston new file mode 100644 index 000000000..df468983a --- /dev/null +++ b/examples/surf_collide/log.22Aug26.mpi_4.piston @@ -0,0 +1,124 @@ +SPARTA (24 Sep 2025) +Running on 4 MPI task(s) +################################################################################ +# a moving piston compressing a gas, with a vanishing outflow wall +# +# Covers surf_collide piston and surf_collide vanish, neither of which appears +# in any other deck. piston drives xlo inward so the gas is compressed and the +# temperature rises; vanish deletes anything reaching xhi, so Np falls. Both +# effects are visible in stats, which is what makes this a test rather than a +# smoke check. +# +# Note: +# - The "comm/sort" option to the "global" command is used to match MPI runs. +# - The "twopass" option is used to match Kokkos runs. +################################################################################ + +seed 12345 +dimension 3 +global gridcut 1.0e-5 comm/sort yes +boundary s s p +create_box 0 0.0001 0 0.0001 0 0.0001 +Created orthogonal box = (0 0 0) to (0.0001 0.0001 0.0001) +create_grid 10 10 10 +WARNING: Could not acquire nearby ghost cells b/c grid partition is not clumped (/home/user/sparta/src/grid.cpp:486) +Created 1000 child grid cells + CPU time = 0.0010845 secs + create/ghost percent = 85.1602 14.8398 +balance_grid rcb part +Balance grid migrated 740 cells + CPU time = 0.000848842 secs + reassign/sort/migrate/ghost percent = 40.6818 0.657602 16.7213 41.9394 + +species air.species N2 N +mixture air N2 N vstream 0.0 0.0 0.0 temp 300.0 +mixture air N2 frac 1.0 +mixture air N frac 0.0 + +global nrho 7.07043E22 fnum 7.07043E6 + +collide vss air air.vss + +surf_collide pist piston 400.0 +surf_collide gone vanish +surf_collide wall specular + +bound_modify xlo collide pist +bound_modify xhi collide gone +bound_modify ylo collide wall +bound_modify yhi collide wall + +create_particles air n 20000 twopass +Created 20000 particles + CPU time = 0.0017963 secs + +compute t temp +stats 50 +stats_style step np nattempt ncoll c_t +timestep 1.0e-9 +run 300 +Memory usage per proc in Mbytes: + particles (ave,min,max) = 1.5625 1.5625 1.5625 + grid (ave,min,max) = 1.51379 1.51379 1.51379 + surf (ave,min,max) = 0 0 0 + modify (ave,min,max) = 0 0 0 + total (ave,min,max) = 3.07629 3.07629 3.07629 +Step Np Natt Ncoll c_t + 0 20000 0 0 301.0787 + 50 17606 545 367 289.64456 + 100 15030 409 263 274.12953 + 150 12562 301 175 254.05321 + 200 10323 192 118 232.94459 + 250 8406 152 104 211.46539 + 300 6931 99 62 195.8943 +Loop time of 0.0274853 on 4 procs for 300 steps with 6931 particles +Performance: 10914.927 timesteps/s, 75.651 Mparticle-step/s + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Move | 0.0088297 | 0.0096265 | 0.010613 | 0.8 | 35.02 +Coll | 0.0065927 | 0.0069242 | 0.0073906 | 0.4 | 25.19 +Sort | 0.0014076 | 0.001527 | 0.0016683 | 0.3 | 5.56 +Comm | 0.0046747 | 0.0048159 | 0.0049142 | 0.1 | 17.52 +Modify | 0 | 0 | 0 | 0.0 | 0.00 +Output | 0.00012169 | 0.00018948 | 0.00036924 | 0.0 | 0.69 +MPI Sync| 0.0028389 | 0.0043758 | 0.0056023 | 1.9 | 15.92 +Other | | 2.644e-05 | | | 0.10 + +Particle moves = 3870498 (3.87M) +Cells touched = 4110885 (4.11M) +Particle comms = 17482 (17.5K) +Boundary collides = 8779 (8.78K) +Boundary exits = 0 (0K) +SurfColl checks = 0 (0K) +SurfColl occurs = 0 (0K) +Surf reactions = 0 (0K) +Collide attempts = 96843 (96.8K) +Collide occurs = 63711 (63.7K) +Reactions = 0 (0K) +Particles stuck = 0 +Axisymm bad moves = 0 + +Particle-moves/CPUsec/proc: 3.52052e+07 +Particle-moves/step: 12901.7 +Cell-touches/particle/step: 1.06211 +Particle comm iterations/step: 1 +Particle fraction communicated: 0.00451673 +Particle fraction colliding with boundary: 0.00226818 +Particle fraction exiting boundary: 0 +Surface-checks/particle/step: 0 +Surface-collisions/particle/step: 0 +Surf-reactions/particle/step: 0 +Collision-attempts/particle/step: 0.0250208 +Collisions/particle/step: 0.0164607 +Reactions/particle/step: 0 + +Particles: 1732.75 ave 1759 max 1678 min +Histogram: 1 0 0 0 0 0 0 1 0 2 +Cells: 250 ave 250 max 250 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +GhostCell: 110 ave 110 max 110 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +EmptyCell: 0 ave 0 max 0 min +Histogram: 4 0 0 0 0 0 0 0 0 0 diff --git a/examples/surf_react_adsorb/in.beam.face.gs b/examples/surf_react_adsorb/in.beam.face.gs index 2c002ffef..63df2bc6c 100644 --- a/examples/surf_react_adsorb/in.beam.face.gs +++ b/examples/surf_react_adsorb/in.beam.face.gs @@ -49,7 +49,7 @@ bound_modify zlo react adsorb_test_gs2 region circle1 cylinder z 0 -10 1 INF INF -fix in1 emit/face/file air zhi data.beam beam_area_1 nevery 100 region circle1 +fix in1 emit/face/file air zhi data.beam beam_area_1 nevery 100 region circle1 twopass ################################################################################################ diff --git a/examples/surf_react_adsorb/in.beam.face.gs_ps b/examples/surf_react_adsorb/in.beam.face.gs_ps index 8285872e9..e6e2e8145 100644 --- a/examples/surf_react_adsorb/in.beam.face.gs_ps +++ b/examples/surf_react_adsorb/in.beam.face.gs_ps @@ -49,7 +49,7 @@ bound_modify zlo react adsorb_test_gs_ps2 region circle1 cylinder z 0 -10 1 INF INF -fix in1 emit/face/file air zhi data.beam beam_area_1 nevery 100 region circle1 +fix in1 emit/face/file air zhi data.beam beam_area_1 nevery 100 region circle1 twopass ################################################################################################ diff --git a/examples/surf_react_adsorb/in.beam.face.ps b/examples/surf_react_adsorb/in.beam.face.ps index b3713f2e4..c33b90ec1 100644 Binary files a/examples/surf_react_adsorb/in.beam.face.ps and b/examples/surf_react_adsorb/in.beam.face.ps differ diff --git a/examples/surf_react_adsorb/in.beam.surf.gs b/examples/surf_react_adsorb/in.beam.surf.gs index 4ae4c3ea0..5fbf02dab 100644 --- a/examples/surf_react_adsorb/in.beam.surf.gs +++ b/examples/surf_react_adsorb/in.beam.surf.gs @@ -49,8 +49,8 @@ surf_modify all collide 1 react adsorb_test_gs2 region circle2 cylinder z 6 -10 1 INF INF region circle3 cylinder z -6 -10 1 INF INF -fix in2 emit/face/file air zhi data.beam beam_area_2 nevery 100 region circle2 -fix in3 emit/face/file air zhi data.beam beam_area_3 nevery 100 region circle3 +fix in2 emit/face/file air zhi data.beam beam_area_2 nevery 100 region circle2 twopass +fix in3 emit/face/file air zhi data.beam beam_area_3 nevery 100 region circle3 twopass ################################################################################################ diff --git a/examples/surf_react_adsorb/in.beam.surf.gs_ps b/examples/surf_react_adsorb/in.beam.surf.gs_ps index cc2b6f81b..4aadd2c61 100644 --- a/examples/surf_react_adsorb/in.beam.surf.gs_ps +++ b/examples/surf_react_adsorb/in.beam.surf.gs_ps @@ -49,8 +49,8 @@ surf_modify all collide 1 react adsorb_test_gs_ps2 region circle2 cylinder z 6 -10 1 INF INF region circle3 cylinder z -6 -10 1 INF INF -fix in2 emit/face/file air zhi data.beam beam_area_2 nevery 100 region circle2 -fix in3 emit/face/file air zhi data.beam beam_area_3 nevery 100 region circle3 +fix in2 emit/face/file air zhi data.beam beam_area_2 nevery 100 region circle2 twopass +fix in3 emit/face/file air zhi data.beam beam_area_3 nevery 100 region circle3 twopass ################################################################################################ diff --git a/examples/surf_react_adsorb/in.beam.surf.ps b/examples/surf_react_adsorb/in.beam.surf.ps index 0f7a9ab7b..ab124b421 100644 Binary files a/examples/surf_react_adsorb/in.beam.surf.ps and b/examples/surf_react_adsorb/in.beam.surf.ps differ diff --git a/examples/surf_react_adsorb/log.11Sep23.mpi_1.beam.face.gs b/examples/surf_react_adsorb/log.11Sep23.mpi_1.beam.face.gs deleted file mode 100644 index 36e866f53..000000000 --- a/examples/surf_react_adsorb/log.11Sep23.mpi_1.beam.face.gs +++ /dev/null @@ -1,237 +0,0 @@ -SPARTA (13 Apr 2023) -Running on 1 MPI task(s) -################################################################################ -# beam of particles striking the surface at an inclined angle -# free molecular flow (no collisions) -# -# Note: -# - The "comm/sort” option to the “global” command is used to match MPI runs. -# The "comm/sort" option should not be used for production runs. -################################################################################ - -seed 123456 -dimension 3 -global gridcut 0.0 comm/sort yes - -boundary oo oo so - - -create_box -11 11 -11 11 0 10 -Created orthogonal box = (-11 -11 0) to (11 11 10) -create_grid 2 2 2 -Created 8 child grid cells - CPU time = 0.000824202 secs - create/ghost percent = 98.1558 1.84421 -balance_grid rcb cell -Balance grid migrated 0 cells - CPU time = 8.06e-05 secs - reassign/sort/migrate/ghost percent = 78.0397 0.248139 16.3772 5.33499 - -global nrho 1e10 fnum 1e6 - -species air.species O CO CO2 O2 C -mixture air O O2 vstream 0 1000 -1000 - -mixture air O frac 1.0 -mixture air CO frac 0.0 -mixture air CO2 frac 0.0 -mixture air C frac 0.0 -mixture air O2 frac 0.0 - - -surf_collide 1 cll 300.0 0.5 0.5 0.5 0.5 - -bound_modify zlo collide 1 - -##################################### SURF REACT ADSORB ###################################### -##################################### FACE/BOUNDARY OPTION ################################### - -#surf_react adsorb_test_gs1 adsorb gs sample-GS_1.surf nsync 1 face 1000 6.022e18 O CO -#bound_modify zlo react adsorb_test_gs1 - - -surf_react adsorb_test_gs2 adsorb gs sample-GS_2.surf nsync 1 face 1000 6.022e18 O CO -bound_modify zlo react adsorb_test_gs2 - -########################## BEAM ############################################################ -# Beam at multiple points so that different processors handle the surface collisions - -region circle1 cylinder z 0 -10 1 -INF INF - -fix in1 emit/face/file air zhi data.beam beam_area_1 nevery 100 region circle1 - -################################################################################################ - -#dump 2 image all 10 image.*.ppm type type pdiam 0.2 surf proc 0.01 size 512 512 zoom 1.75 gline no 0.005 -#dump_modify 2 pad 4 - -timestep 0.0001 - -stats 10 -stats_style step cpu np nattempt ncoll nscoll nscheck -run 1000 -Memory usage per proc in Mbytes: - particles (ave,min,max) = 0 0 0 - grid (ave,min,max) = 1.51379 1.51379 1.51379 - surf (ave,min,max) = 0 0 0 - total (ave,min,max) = 1.51379 1.51379 1.51379 -Step CPU Np Natt Ncoll Nscoll Nscheck - 0 0 0 0 0 0 0 - 10 6.5e-06 0 0 0 0 0 - 20 2.59e-05 0 0 0 0 0 - 30 4.45e-05 0 0 0 0 0 - 40 6.3e-05 0 0 0 0 0 - 50 8.15e-05 0 0 0 0 0 - 60 9.99e-05 0 0 0 0 0 - 70 0.0001184 0 0 0 0 0 - 80 0.0001371 0 0 0 0 0 - 90 0.0001556 0 0 0 0 0 - 100 0.001853604 3150 0 0 0 0 - 110 0.006447714 3150 0 0 0 0 - 120 0.01446033 3150 0 0 0 0 - 130 0.014801531 3150 0 0 0 0 - 140 0.015110132 3150 0 0 0 0 - 150 0.015438932 3150 0 0 0 0 - 160 0.015746633 3150 0 0 0 0 - 170 0.016049034 3150 0 0 0 0 - 180 0.016351534 3150 0 0 0 0 - 190 0.016659335 3150 0 0 0 0 - 200 0.018635039 3230 0 0 0 0 - 210 0.026457155 3204 0 0 0 0 - 220 0.026798556 3204 0 0 0 0 - 230 0.027109757 3204 0 0 0 0 - 240 0.027421957 3204 0 0 0 0 - 250 0.027755358 3204 0 0 0 0 - 260 0.028069859 3204 0 0 0 0 - 270 0.028379959 3204 0 0 0 0 - 280 0.02869016 3204 0 0 0 0 - 290 0.028999361 3204 0 0 0 0 - 300 0.030921165 3299 0 0 0 0 - 310 0.031252265 3272 0 0 0 0 - 320 0.031568866 3272 0 0 0 0 - 330 0.031886267 3271 0 0 0 0 - 340 0.032204767 3270 0 0 0 0 - 350 0.032551368 3268 0 0 0 0 - 360 0.032867769 3267 0 0 0 0 - 370 0.03318667 3266 0 0 0 0 - 380 0.03350697 3264 0 0 0 0 - 390 0.042504789 3257 0 0 0 0 - 400 0.044470893 3379 0 0 0 0 - 410 0.044809294 3346 0 0 0 0 - 420 0.045133295 3345 0 0 0 0 - 430 0.045458895 3343 0 0 0 0 - 440 0.045782896 3339 0 0 0 0 - 450 0.046141797 3330 0 0 0 0 - 460 0.046465297 3322 0 0 0 0 - 470 0.046785298 3315 0 0 0 0 - 480 0.047103499 3312 0 0 0 0 - 490 0.047427399 3306 0 0 0 0 - 500 0.049336903 3384 0 0 0 0 - 510 0.049671804 3343 0 0 0 0 - 520 0.050009405 3337 0 0 0 0 - 530 0.050341405 3328 0 0 0 0 - 540 0.050666406 3318 0 0 0 0 - 550 0.051012807 3312 0 0 0 0 - 560 0.058534723 3302 0 0 0 0 - 570 0.058889423 3292 0 0 0 0 - 580 0.059207024 3285 0 0 0 0 - 590 0.059525025 3278 0 0 0 0 - 600 0.061415129 3356 0 0 0 0 - 610 0.061746629 3317 0 0 0 0 - 620 0.06206613 3311 0 0 0 0 - 630 0.062394031 3308 0 0 0 0 - 640 0.062716831 3303 0 0 0 0 - 650 0.063065232 3297 0 0 0 0 - 660 0.063383533 3288 0 0 0 0 - 670 0.063701833 3280 0 0 0 0 - 680 0.064023734 3274 0 0 0 0 - 690 0.064341035 3268 0 0 0 0 - 700 0.066226739 3374 0 0 0 0 - 710 0.074477956 3340 0 0 0 0 - 720 0.074831657 3335 0 0 0 0 - 730 0.075152857 3326 0 0 0 0 - 740 0.075472158 3320 0 0 0 0 - 750 0.075819459 3314 0 0 0 0 - 760 0.076137559 3310 0 0 0 0 - 770 0.07645746 3303 0 0 0 0 - 780 0.076776961 3295 0 0 0 0 - 790 0.077094961 3286 0 0 0 0 - 800 0.078995565 3406 0 0 0 0 - 810 0.079349466 3368 0 0 0 0 - 820 0.079675167 3356 0 0 0 0 - 830 0.079996467 3351 0 0 0 0 - 840 0.080318768 3329 0 0 0 0 - 850 0.080671669 3321 0 0 0 0 - 860 0.08099157 3319 0 0 0 0 - 870 0.08131047 3306 0 0 0 0 - 880 0.090460689 3299 0 0 0 0 - 890 0.09081429 3294 0 0 0 0 - 900 0.092723294 3385 0 0 0 0 - 910 0.093055795 3350 0 0 0 0 - 920 0.093378695 3341 0 0 0 0 - 930 0.093704496 3333 0 0 0 0 - 940 0.094029297 3326 0 0 0 0 - 950 0.094386598 3318 0 0 0 0 - 960 0.094708598 3310 0 0 0 0 - 970 0.095028599 3299 0 0 0 0 - 980 0.0953473 3293 0 0 0 0 - 990 0.095664 3286 0 0 0 0 - 1000 0.097552104 3388 0 0 0 0 -Loop time of 0.0975686 on 1 procs for 1000 steps with 3388 particles - -MPI task timing breakdown: -Section | min time | avg time | max time |%varavg| %total ---------------------------------------------------------------- -Move | 0.034708 | 0.034708 | 0.034708 | 0.0 | 35.57 -Coll | 0 | 0 | 0 | 0.0 | 0.00 -Sort | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.0003552 | 0.0003552 | 0.0003552 | 0.0 | 0.36 -Modify | 0.0084601 | 0.0084601 | 0.0084601 | 0.0 | 8.67 -Output | 0.053829 | 0.053829 | 0.053829 | 0.0 | 55.17 -Other | | 0.0002159 | | | 0.22 - -Particle moves = 2982819 (2.98M) -Cells touched = 3026072 (3.03M) -Particle comms = 0 (0K) -Boundary collides = 645 (0.645K) -Boundary exits = 432 (0.432K) -SurfColl checks = 0 (0K) -SurfColl occurs = 0 (0K) -Surf reactions = 28160 (28.2K) -Collide attempts = 0 (0K) -Collide occurs = 0 (0K) -Reactions = 0 (0K) -Particles stuck = 0 -Axisymm bad moves = 0 - -Particle-moves/CPUsec/proc: 3.05715e+07 -Particle-moves/step: 2982.82 -Cell-touches/particle/step: 1.0145 -Particle comm iterations/step: 1 -Particle fraction communicated: 0 -Particle fraction colliding with boundary: 0.000216238 -Particle fraction exiting boundary: 0.000144829 -Surface-checks/particle/step: 0 -Surface-collisions/particle/step: 0 -Surf-reactions/particle/step: 0.00944073 -Collision-attempts/particle/step: 0 -Collisions/particle/step: 0 -Reactions/particle/step: 0 - -Surface reaction tallies: - id adsorb_test_gs2 style adsorb #-of-reactions 9 - reaction all: 28160 - reaction O(g) --> O(s): 20986 - reaction O(g) + O(s) --> CO2(g): 2 - reaction O(g) --> CO(s): 6529 - reaction O(g) --> CO(g): 638 - reaction O(g) + O(s) --> O(g) + O(g): 5 - -Particles: 3388 ave 3388 max 3388 min -Histogram: 1 0 0 0 0 0 0 0 0 0 -Cells: 8 ave 8 max 8 min -Histogram: 1 0 0 0 0 0 0 0 0 0 -GhostCell: 0 ave 0 max 0 min -Histogram: 1 0 0 0 0 0 0 0 0 0 -EmptyCell: 0 ave 0 max 0 min -Histogram: 1 0 0 0 0 0 0 0 0 0 diff --git a/examples/surf_react_adsorb/log.11Sep23.mpi_1.beam.face.gs_ps b/examples/surf_react_adsorb/log.11Sep23.mpi_1.beam.face.gs_ps deleted file mode 100644 index d0ce8d0a8..000000000 --- a/examples/surf_react_adsorb/log.11Sep23.mpi_1.beam.face.gs_ps +++ /dev/null @@ -1,240 +0,0 @@ -SPARTA (13 Apr 2023) -Running on 1 MPI task(s) -################################################################################ -# beam of particles striking the surface at an inclined angle -# free molecular flow (no collisions) -# -# Note: -# - The "comm/sort” option to the “global” command is used to match MPI runs. -# The "comm/sort" option should not be used for production runs. -################################################################################ - -seed 123456 -dimension 3 -global gridcut 0.0 comm/sort yes - -boundary oo oo so - - -create_box -11 11 -11 11 0 10 -Created orthogonal box = (-11 -11 0) to (11 11 10) -create_grid 2 2 2 -Created 8 child grid cells - CPU time = 0.000823302 secs - create/ghost percent = 97.3157 2.68431 -balance_grid rcb cell -Balance grid migrated 0 cells - CPU time = 7.79e-05 secs - reassign/sort/migrate/ghost percent = 74.4544 0.256739 19.3838 5.90501 - -global nrho 1e10 fnum 1e6 - -species air.species O CO CO2 O2 C -mixture air O O2 vstream 0 1000 -1000 - -mixture air O frac 1.0 -mixture air CO frac 0.0 -mixture air CO2 frac 0.0 -mixture air C frac 0.0 -mixture air O2 frac 0.0 - - -surf_collide 1 cll 300.0 0.5 0.5 0.5 0.5 - -bound_modify zlo collide 1 - -##################################### SURF REACT ADSORB ###################################### -##################################### FACE/BOUNDARY OPTION ################################### - -#surf_react adsorb_test_gs_ps1 adsorb gs/ps sample-GS_1.surf sample-PS_1.surf nsync 1 face 1000 6.022e18 O CO -#bound_modify zlo react adsorb_test_gs_ps1 - - -surf_react adsorb_test_gs_ps2 adsorb gs/ps sample-GS_2.surf sample-PS_2.surf nsync 1 face 1000 6.022e18 O CO -bound_modify zlo react adsorb_test_gs_ps2 - -########################## BEAM ############################################################ -# Beam at multiple points so that different processors handle the surface collisions - -region circle1 cylinder z 0 -10 1 -INF INF - -fix in1 emit/face/file air zhi data.beam beam_area_1 nevery 100 region circle1 - -################################################################################################ - -#dump 2 image all 10 image.*.ppm type type pdiam 0.2 surf proc 0.01 size 512 512 zoom 1.75 gline no 0.005 -#dump_modify 2 pad 4 - -timestep 0.0001 - -stats 10 -stats_style step cpu np nattempt ncoll nscoll nscheck -run 1000 -Memory usage per proc in Mbytes: - particles (ave,min,max) = 0 0 0 - grid (ave,min,max) = 1.51379 1.51379 1.51379 - surf (ave,min,max) = 0 0 0 - total (ave,min,max) = 1.51379 1.51379 1.51379 -Step CPU Np Natt Ncoll Nscoll Nscheck - 0 0 0 0 0 0 0 - 10 1.09e-05 0 0 0 0 0 - 20 3.28e-05 0 0 0 0 0 - 30 5.36e-05 0 0 0 0 0 - 40 7.43e-05 0 0 0 0 0 - 50 9.5001e-05 0 0 0 0 0 - 60 0.000115801 0 0 0 0 0 - 70 0.000137101 0 0 0 0 0 - 80 0.000158101 0 0 0 0 0 - 90 0.000178701 0 0 0 0 0 - 100 0.001782004 3130 0 0 0 0 - 110 0.002641306 3131 0 0 0 0 - 120 0.002965107 3131 0 0 0 0 - 130 0.006571814 3131 0 0 0 0 - 140 0.006895815 3131 0 0 0 0 - 150 0.007223515 3131 0 0 0 0 - 160 0.007527116 3131 0 0 0 0 - 170 0.007829717 3131 0 0 0 0 - 180 0.008134717 3132 0 0 0 0 - 190 0.008441618 3134 0 0 0 0 - 200 0.010624923 3858 0 0 0 0 - 210 0.015487933 5794 0 0 0 0 - 220 0.016195834 6003 0 0 0 0 - 230 0.016804136 6028 0 0 0 0 - 240 0.01885954 5983 0 0 0 0 - 250 0.019476741 5913 0 0 0 0 - 260 0.020039142 5827 0 0 0 0 - 270 0.026828756 5745 0 0 0 0 - 280 0.027416358 5672 0 0 0 0 - 290 0.027958459 5585 0 0 0 0 - 300 0.030467164 6566 0 0 0 0 - 310 0.031768367 7966 0 0 0 0 - 320 0.032639069 8039 0 0 0 0 - 330 0.035059574 7885 0 0 0 0 - 340 0.035832875 7708 0 0 0 0 - 350 0.04302669 7520 0 0 0 0 - 360 0.043764592 7324 0 0 0 0 - 370 0.044459093 7129 0 0 0 0 - 380 0.045134495 6936 0 0 0 0 - 390 0.045789596 6719 0 0 0 0 - 400 0.047997901 6801 0 0 0 0 - 410 0.049424304 8493 0 0 0 0 - 420 0.059301024 8759 0 0 0 0 - 430 0.060196726 8583 0 0 0 0 - 440 0.061027128 8313 0 0 0 0 - 450 0.06185433 8088 0 0 0 0 - 460 0.062633131 7823 0 0 0 0 - 470 0.063381633 7599 0 0 0 0 - 480 0.064101435 7351 0 0 0 0 - 490 0.064797236 7125 0 0 0 0 - 500 0.067234341 7756 0 0 0 0 - 510 0.075681459 9086 0 0 0 0 - 520 0.076708661 9028 0 0 0 0 - 530 0.077601763 8815 0 0 0 0 - 540 0.078451765 8571 0 0 0 0 - 550 0.079306166 8295 0 0 0 0 - 560 0.080089568 8056 0 0 0 0 - 570 0.08085467 7816 0 0 0 0 - 580 0.091004791 7559 0 0 0 0 - 590 0.091751592 7309 0 0 0 0 - 600 0.094337498 8108 0 0 0 0 - 610 0.095752301 9318 0 0 0 0 - 620 0.096724703 9200 0 0 0 0 - 630 0.097617505 8966 0 0 0 0 - 640 0.098474806 8715 0 0 0 0 - 650 0.099341708 8452 0 0 0 0 - 660 0.10705992 8194 0 0 0 0 - 670 0.10785923 7952 0 0 0 0 - 680 0.10861213 7696 0 0 0 0 - 690 0.10934033 7428 0 0 0 0 - 700 0.11187303 8131 0 0 0 0 - 710 0.11333874 9402 0 0 0 0 - 720 0.12326916 9360 0 0 0 0 - 730 0.12422586 9111 0 0 0 0 - 740 0.12509216 8851 0 0 0 0 - 750 0.12597026 8561 0 0 0 0 - 760 0.12678697 8299 0 0 0 0 - 770 0.12757987 8018 0 0 0 0 - 780 0.12834357 7762 0 0 0 0 - 790 0.12907797 7527 0 0 0 0 - 800 0.13162388 8219 0 0 0 0 - 810 0.13951319 8988 0 0 0 0 - 820 0.14046149 8834 0 0 0 0 - 830 0.1414879 9020 0 0 0 0 - 840 0.1424084 8860 0 0 0 0 - 850 0.143313 8619 0 0 0 0 - 860 0.1441369 8344 0 0 0 0 - 870 0.15106802 8081 0 0 0 0 - 880 0.15185792 7835 0 0 0 0 - 890 0.15260192 7580 0 0 0 0 - 900 0.15492253 7649 0 0 0 0 - 910 0.15647893 9408 0 0 0 0 - 920 0.16733205 9483 0 0 0 0 - 930 0.16829105 9264 0 0 0 0 - 940 0.16917975 8959 0 0 0 0 - 950 0.17008126 8675 0 0 0 0 - 960 0.17091416 8371 0 0 0 0 - 970 0.17170796 8084 0 0 0 0 - 980 0.17247366 7830 0 0 0 0 - 990 0.17321946 7579 0 0 0 0 - 1000 0.17575987 8301 0 0 0 0 -Loop time of 0.175787 on 1 procs for 1000 steps with 8301 particles - -MPI task timing breakdown: -Section | min time | avg time | max time |%varavg| %total ---------------------------------------------------------------- -Move | 0.069387 | 0.069387 | 0.069387 | 0.0 | 39.47 -Coll | 0 | 0 | 0 | 0.0 | 0.00 -Sort | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.0005442 | 0.0005442 | 0.0005442 | 0.0 | 0.31 -Modify | 0.0076668 | 0.0076668 | 0.0076668 | 0.0 | 4.36 -Output | 0.091595 | 0.091595 | 0.091595 | 0.0 | 52.11 -Other | | 0.006593 | | | 3.75 - -Particle moves = 6656046 (6.66M) -Cells touched = 6721993 (6.72M) -Particle comms = 0 (0K) -Boundary collides = 629 (0.629K) -Boundary exits = 19114 (19.1K) -SurfColl checks = 0 (0K) -SurfColl occurs = 0 (0K) -Surf reactions = 28216 (28.2K) -Collide attempts = 0 (0K) -Collide occurs = 0 (0K) -Reactions = 0 (0K) -Particles stuck = 0 -Axisymm bad moves = 0 - -Particle-moves/CPUsec/proc: 3.78643e+07 -Particle-moves/step: 6656.05 -Cell-touches/particle/step: 1.00991 -Particle comm iterations/step: 1 -Particle fraction communicated: 0 -Particle fraction colliding with boundary: 9.45005e-05 -Particle fraction exiting boundary: 0.00287167 -Surface-checks/particle/step: 0 -Surface-collisions/particle/step: 0 -Surf-reactions/particle/step: 0.00423915 -Collision-attempts/particle/step: 0 -Collisions/particle/step: 0 -Reactions/particle/step: 0 - -Surface reaction tallies: - id adsorb_test_gs_ps2 style adsorb #-of-reactions 14 - reaction all: 52305 - reaction O(g) --> O(s): 21020 - reaction O(g) --> CO(s): 6567 - reaction O(g) --> CO(g): 629 - reaction O(s) --> O(g): 15091 - reaction CO(s) --> CO(g): 6379 - reaction 2O(s) + C(b) --> CO2(g): 2110 - reaction O(s) + C(b) --> CO(s): 493 - reaction C(b) --> C(g): 16 - -Particles: 8301 ave 8301 max 8301 min -Histogram: 1 0 0 0 0 0 0 0 0 0 -Cells: 8 ave 8 max 8 min -Histogram: 1 0 0 0 0 0 0 0 0 0 -GhostCell: 0 ave 0 max 0 min -Histogram: 1 0 0 0 0 0 0 0 0 0 -EmptyCell: 0 ave 0 max 0 min -Histogram: 1 0 0 0 0 0 0 0 0 0 diff --git a/examples/surf_react_adsorb/log.11Sep23.mpi_1.beam.face.ps b/examples/surf_react_adsorb/log.11Sep23.mpi_1.beam.face.ps deleted file mode 100644 index 1926183fd..000000000 Binary files a/examples/surf_react_adsorb/log.11Sep23.mpi_1.beam.face.ps and /dev/null differ diff --git a/examples/surf_react_adsorb/log.11Sep23.mpi_1.beam.surf.gs b/examples/surf_react_adsorb/log.11Sep23.mpi_1.beam.surf.gs deleted file mode 100644 index 84a0faabd..000000000 --- a/examples/surf_react_adsorb/log.11Sep23.mpi_1.beam.surf.gs +++ /dev/null @@ -1,257 +0,0 @@ -SPARTA (13 Apr 2023) -Running on 1 MPI task(s) -################################################################################ -# beam of particles striking the surface at an inclined angle -# free molecular flow (no collisions) -# -# Note: -# - The "comm/sort” option to the “global” command is used to match MPI runs. -# The "comm/sort" option should not be used for production runs. -################################################################################ - -seed 123456 -dimension 3 -global gridcut 0.0 comm/sort yes - -boundary oo oo oo - - -create_box -11 11 -11 11 0 10 -Created orthogonal box = (-11 -11 0) to (11 11 10) -create_grid 2 2 2 -Created 8 child grid cells - CPU time = 0.000805202 secs - create/ghost percent = 97.69 2.30998 -balance_grid rcb cell -Balance grid migrated 0 cells - CPU time = 0.0001155 secs - reassign/sort/migrate/ghost percent = 83.2035 0.4329 11.9481 4.41558 - -global nrho 1e10 fnum 1e6 - -species air.species O CO CO2 O2 C -mixture air O O2 vstream 0 1000 -1000 - -mixture air O frac 1.0 -mixture air CO frac 0.0 -mixture air CO2 frac 0.0 -mixture air C frac 0.0 -mixture air O2 frac 0.0 - - -surf_collide 1 cll 300.0 0.5 0.5 0.5 0.5 - -read_surf base_plate.surf - 8 points - 12 triangles - -11 11 xlo xhi - -11 11 ylo yhi - 0 1 zlo zhi - 1 min triangle edge length - 11 min triangle area - 4 0 = cells overlapping surfs, overlap cells with unmarked corner pts - 4 0 4 = cells outside/inside/overlapping surfs - 4 = surf cells with 1,2,etc splits - 4356 4356 = cell-wise and global flow volume - CPU time = 0.000810501 secs - read/check/sort/surf2grid/ghost/inout/particle percent = 15.7557 10.2776 0.111042 68.3282 5.52745 6.16915 0.0246761 - surf2grid time = 0.000553801 secs - map/comm1/comm2/comm3/comm4/split percent = 14.3012 2.61827 1.76959 1.37233 5.03791 74.2326 - -##################################### SURF REACT ADSORB ###################################### -##################################### SURF OPTION ############################################ - -#surf_react adsorb_test_gs1 adsorb gs sample-GS_1.surf nsync 1 surf 1000 6.022e18 O CO -#surf_modify all collide 1 react adsorb_test_gs1 - -surf_react adsorb_test_gs2 adsorb gs sample-GS_2.surf nsync 1 surf 1000 6.022e18 O CO -surf_modify all collide 1 react adsorb_test_gs2 - -########################## BEAM ############################################################ -# Beam at multiple points so that different processors handle the surface collisions - -region circle2 cylinder z 6 -10 1 -INF INF -region circle3 cylinder z -6 -10 1 -INF INF - -fix in2 emit/face/file air zhi data.beam beam_area_2 nevery 100 region circle2 -fix in3 emit/face/file air zhi data.beam beam_area_3 nevery 100 region circle3 - -################################################################################################ - -#dump 2 image all 10 image.*.ppm type type pdiam 0.2 surf proc 0.01 size 512 512 zoom 1.75 gline no 0.005 -#dump_modify 2 pad 4 - -timestep 0.0001 - -stats 10 -stats_style step cpu np nattempt ncoll nscoll nscheck -run 1000 -Memory usage per proc in Mbytes: - particles (ave,min,max) = 0 0 0 - grid (ave,min,max) = 1.51379 1.51379 1.51379 - surf (ave,min,max) = 0.00151062 0.00151062 0.00151062 - total (ave,min,max) = 1.5153 1.5153 1.5153 -Step CPU Np Natt Ncoll Nscoll Nscheck - 0 0 0 0 0 0 0 - 10 1.35e-05 0 0 0 0 0 - 20 3.69e-05 0 0 0 0 0 - 30 5.95e-05 0 0 0 0 0 - 40 8.19e-05 0 0 0 0 0 - 50 0.0001042 0 0 0 0 0 - 60 0.0001268 0 0 0 0 0 - 70 0.0001492 0 0 0 0 0 - 80 0.0001717 0 0 0 0 0 - 90 0.0002046 0 0 0 0 0 - 100 0.002523905 6218 0 0 0 0 - 110 0.005990912 6218 0 0 0 0 - 120 0.006741314 6218 0 0 0 0 - 130 0.010045721 6218 0 0 0 0 - 140 0.010795622 6218 0 0 0 0 - 150 0.011928624 6218 0 0 0 49504 - 160 0.017191236 6218 0 0 0 49744 - 170 0.029204661 6218 0 0 0 49744 - 180 0.033228969 6218 0 0 0 49744 - 190 0.047002598 189 0 0 6134 50584 - 200 0.054774514 6432 0 0 0 1088 - 210 0.055652416 6432 0 0 0 1072 - 220 0.056486018 6432 0 0 0 1072 - 230 0.05733512 6432 0 0 0 1064 - 240 0.058170921 6432 0 0 0 1056 - 250 0.059418424 6432 0 0 0 51168 - 260 0.069668845 6432 0 0 0 51320 - 270 0.073837054 6430 0 0 0 51224 - 280 0.085346578 6425 0 0 0 51112 - 290 0.09121159 326 0 0 6211 51920 - 300 0.10286171 6607 0 0 0 1736 - 310 0.10380942 6604 0 0 0 1640 - 320 0.10471062 6601 0 0 0 1608 - 330 0.10560922 6597 0 0 0 1584 - 340 0.10650222 6589 0 0 0 1568 - 350 0.11454564 6588 0 0 0 51824 - 360 0.11880265 6579 0 0 0 51944 - 370 0.12299696 6573 0 0 0 51816 - 380 0.13342078 6563 0 0 0 51680 - 390 0.14714221 423 0 0 6240 52480 - 400 0.15486322 6636 0 0 0 2104 - 410 0.15583613 6622 0 0 0 1984 - 420 0.15676283 6607 0 0 0 1920 - 430 0.15770493 6592 0 0 0 1840 - 440 0.15862113 6580 0 0 0 1768 - 450 0.15993013 6563 0 0 0 51496 - 460 0.17346536 6548 0 0 0 51536 - 470 0.17778257 6535 0 0 0 51392 - 480 0.18192478 6526 0 0 0 51232 - 490 0.19508461 447 0 0 6155 51728 - 500 0.20115492 6716 0 0 0 1944 - 510 0.20213502 6705 0 0 0 1824 - 520 0.21017584 6690 0 0 0 1744 - 530 0.21113124 6679 0 0 0 1672 - 540 0.21204864 6665 0 0 0 1560 - 550 0.21336145 6652 0 0 0 51832 - 560 0.22558837 6638 0 0 0 51992 - 570 0.23039828 6625 0 0 0 51864 - 580 0.23459479 6609 0 0 0 51704 - 590 0.24720172 469 0 0 6218 52312 - 600 0.25888184 6764 0 0 0 1928 - 610 0.25986814 6747 0 0 0 1840 - 620 0.26080475 6739 0 0 0 1744 - 630 0.26174495 6722 0 0 0 1704 - 640 0.26267275 6706 0 0 0 1600 - 650 0.26398915 6696 0 0 0 52296 - 660 0.26827436 6689 0 0 0 52336 - 670 0.28147199 6671 0 0 0 52176 - 680 0.2857703 6657 0 0 0 52016 - 690 0.29955713 472 0 0 6271 52688 - 700 0.30692404 6701 0 0 0 1920 - 710 0.30790284 6685 0 0 0 1872 - 720 0.30883435 6667 0 0 0 1760 - 730 0.30976765 6649 0 0 0 1648 - 740 0.31068545 6630 0 0 0 1552 - 750 0.31856367 6622 0 0 0 51616 - 760 0.32278258 6607 0 0 0 51752 - 770 0.32697198 6593 0 0 0 51584 - 780 0.33746901 6582 0 0 0 51496 - 790 0.35107853 450 0 0 6205 52120 - 800 0.35882325 6628 0 0 0 1928 - 810 0.35977655 6609 0 0 0 1832 - 820 0.36069375 6598 0 0 0 1744 - 830 0.36162096 6579 0 0 0 1640 - 840 0.36253066 6568 0 0 0 1528 - 850 0.36382956 6549 0 0 0 51080 - 860 0.37737839 6530 0 0 0 51160 - 870 0.3815309 6511 0 0 0 51016 - 880 0.38565481 6499 0 0 0 50904 - 890 0.39902733 449 0 0 6140 51632 - 900 0.40682285 6634 0 0 0 1920 - 910 0.40778605 6621 0 0 0 1824 - 920 0.40870406 6603 0 0 0 1720 - 930 0.40962256 6587 0 0 0 1640 - 940 0.41815147 6571 0 0 0 1544 - 950 0.41949478 6556 0 0 0 51280 - 960 0.42370879 6541 0 0 0 51384 - 970 0.4278607 6529 0 0 0 51264 - 980 0.44137262 6519 0 0 0 51136 - 990 0.44724114 470 0 0 6165 52048 - 1000 0.45886616 6662 0 0 0 2280 -Loop time of 0.458925 on 1 procs for 1000 steps with 6662 particles - -MPI task timing breakdown: -Section | min time | avg time | max time |%varavg| %total ---------------------------------------------------------------- -Move | 0.21823 | 0.21823 | 0.21823 | 0.0 | 47.55 -Coll | 0 | 0 | 0 | 0.0 | 0.00 -Sort | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.0003319 | 0.0003319 | 0.0003319 | 0.0 | 0.07 -Modify | 0.014124 | 0.014124 | 0.014124 | 0.0 | 3.08 -Output | 0.22549 | 0.22549 | 0.22549 | 0.0 | 49.13 -Other | | 0.000748 | | | 0.16 - -Particle moves = 5397859 (5.4M) -Cells touched = 5456087 (5.46M) -Particle comms = 0 (0K) -Boundary collides = 0 (0K) -Boundary exits = 938 (0.938K) -SurfColl checks = 19767352 (19.8M) -SurfColl occurs = 56634 (56.6K) -Surf reactions = 56634 (56.6K) -Collide attempts = 0 (0K) -Collide occurs = 0 (0K) -Reactions = 0 (0K) -Particles stuck = 0 -Axisymm bad moves = 0 - -Particle-moves/CPUsec/proc: 1.1762e+07 -Particle-moves/step: 5397.86 -Cell-touches/particle/step: 1.01079 -Particle comm iterations/step: 1 -Particle fraction communicated: 0 -Particle fraction colliding with boundary: 0 -Particle fraction exiting boundary: 0.000173773 -Surface-checks/particle/step: 3.66207 -Surface-collisions/particle/step: 0.0104919 -Surf-reactions/particle/step: 0.0104919 -Collision-attempts/particle/step: 0 -Collisions/particle/step: 0 -Reactions/particle/step: 0 - -Surface reaction tallies: - id adsorb_test_gs2 style adsorb #-of-reactions 9 - reaction all: 56634 - reaction O(g) --> O(s): 42304 - reaction O(g) + O(s) --> CO2(g): 8 - reaction O(g) --> CO(s): 13008 - reaction O(g) --> CO(g): 1292 - reaction O(g) + O(s) --> O(g) + O(g): 22 - -Particles: 6662 ave 6662 max 6662 min -Histogram: 1 0 0 0 0 0 0 0 0 0 -Cells: 8 ave 8 max 8 min -Histogram: 1 0 0 0 0 0 0 0 0 0 -GhostCell: 0 ave 0 max 0 min -Histogram: 1 0 0 0 0 0 0 0 0 0 -EmptyCell: 0 ave 0 max 0 min -Histogram: 1 0 0 0 0 0 0 0 0 0 -Surfs: 12 ave 12 max 12 min -Histogram: 1 0 0 0 0 0 0 0 0 0 -GhostSurf: 0 ave 0 max 0 min -Histogram: 1 0 0 0 0 0 0 0 0 0 diff --git a/examples/surf_react_adsorb/log.11Sep23.mpi_1.beam.surf.gs_ps b/examples/surf_react_adsorb/log.11Sep23.mpi_1.beam.surf.gs_ps deleted file mode 100644 index 644800bd8..000000000 --- a/examples/surf_react_adsorb/log.11Sep23.mpi_1.beam.surf.gs_ps +++ /dev/null @@ -1,261 +0,0 @@ -SPARTA (13 Apr 2023) -Running on 1 MPI task(s) -################################################################################ -# beam of particles striking the surface at an inclined angle -# free molecular flow (no collisions) -# -# Note: -# - The "comm/sort” option to the “global” command is used to match MPI runs. -# The "comm/sort" option should not be used for production runs. -################################################################################ - -seed 123456 -dimension 3 -global gridcut 0.0 comm/sort yes - -boundary oo oo oo - - -create_box -11 11 -11 11 0 10 -Created orthogonal box = (-11 -11 0) to (11 11 10) -create_grid 2 2 2 -Created 8 child grid cells - CPU time = 0.000810401 secs - create/ghost percent = 97.8282 2.17176 -balance_grid rcb cell -Balance grid migrated 0 cells - CPU time = 7.4e-05 secs - reassign/sort/migrate/ghost percent = 74.8649 0.27027 18.7838 6.08108 - -global nrho 1e10 fnum 1e6 - -species air.species O CO CO2 O2 C -mixture air O O2 vstream 0 1000 -1000 - -mixture air O frac 1.0 -mixture air CO frac 0.0 -mixture air CO2 frac 0.0 -mixture air C frac 0.0 -mixture air O2 frac 0.0 - - -surf_collide 1 cll 300.0 0.5 0.5 0.5 0.5 - -read_surf base_plate.surf - 8 points - 12 triangles - -11 11 xlo xhi - -11 11 ylo yhi - 0 1 zlo zhi - 1 min triangle edge length - 11 min triangle area - 4 0 = cells overlapping surfs, overlap cells with unmarked corner pts - 4 0 4 = cells outside/inside/overlapping surfs - 4 = surf cells with 1,2,etc splits - 4356 4356 = cell-wise and global flow volume - CPU time = 0.000522701 secs - read/check/sort/surf2grid/ghost/inout/particle percent = 24.163 16.2236 0.286971 51.8461 7.48038 9.5083 0.0191314 - surf2grid time = 0.000271 secs - map/comm1/comm2/comm3/comm4/split percent = 30 5.38745 3.80074 2.54613 15.2399 41.6236 - -##################################### SURF REACT ADSORB ###################################### -##################################### SURF OPTION ############################################ - -#surf_react adsorb_test_gs_ps1 adsorb gs/ps sample-GS_1.surf sample-PS_1.surf nsync 1 surf 1000 6.022e18 O CO -#surf_modify all collide 1 react adsorb_test_gs_ps1 - -surf_react adsorb_test_gs_ps2 adsorb gs/ps sample-GS_2.surf sample-PS_2.surf nsync 1 surf 1000 6.022e18 O CO -surf_modify all collide 1 react adsorb_test_gs_ps2 - -########################## BEAM ############################################################ -# Beam at multiple points so that different processors handle the surface collisions - -region circle2 cylinder z 6 -10 1 -INF INF -region circle3 cylinder z -6 -10 1 -INF INF - -fix in2 emit/face/file air zhi data.beam beam_area_2 nevery 100 region circle2 -fix in3 emit/face/file air zhi data.beam beam_area_3 nevery 100 region circle3 - -################################################################################################ - -#dump 2 image all 10 image.*.ppm type type pdiam 0.2 surf proc 0.01 size 512 512 zoom 1.75 gline no 0.005 -#dump_modify 2 pad 4 - -timestep 0.0001 - -stats 10 -stats_style step cpu np nattempt ncoll nscoll nscheck -run 1000 -Memory usage per proc in Mbytes: - particles (ave,min,max) = 0 0 0 - grid (ave,min,max) = 1.51379 1.51379 1.51379 - surf (ave,min,max) = 0.00151062 0.00151062 0.00151062 - total (ave,min,max) = 1.5153 1.5153 1.5153 -Step CPU Np Natt Ncoll Nscoll Nscheck - 0 0 0 0 0 0 0 - 10 4.52e-05 0 0 0 0 0 - 20 8.65e-05 0 0 0 0 0 - 30 0.0001257 0 0 0 0 0 - 40 0.0001649 0 0 0 0 0 - 50 0.000204 0 0 0 0 0 - 60 0.000243 0 0 0 0 0 - 70 0.000282 0 0 0 0 0 - 80 0.0003311 0 0 0 0 0 - 90 0.0003712 0 0 0 0 0 - 100 0.006774315 6293 0 0 0 0 - 110 0.007576117 6295 0 0 0 16 - 120 0.008345119 6295 0 0 0 24 - 130 0.009110921 6296 0 0 0 16 - 140 0.009897822 6295 0 0 0 16 - 150 0.014642333 6296 0 0 0 50176 - 160 0.026808161 6297 0 0 0 50368 - 170 0.030957671 6297 0 0 0 50368 - 180 0.03508678 6298 0 0 0 50368 - 190 0.04810771 1581 0 0 6214 51208 - 200 0.063334045 11544 0 0 0 41328 - 210 0.068150256 11985 0 0 0 45864 - 220 0.082076988 11961 0 0 0 45752 - 230 0.086677599 11869 0 0 0 44496 - 240 0.097805725 11718 0 0 0 41312 - 250 0.11001185 11588 0 0 0 86776 - 260 0.12529299 11451 0 0 0 82064 - 270 0.1321743 11265 0 0 0 76984 - 280 0.14960134 11068 0 0 0 72848 - 290 0.1746178 6777 0 0 6166 70600 - 300 0.18089822 14119 0 0 0 44048 - 310 0.19497725 14692 0 0 0 48024 - 320 0.20063426 14768 0 0 0 49240 - 330 0.21490959 14536 0 0 0 47808 - 340 0.22663182 14165 0 0 0 44448 - 350 0.23998095 13822 0 0 0 91072 - 360 0.25598879 13388 0 0 0 87040 - 370 0.26364121 13030 0 0 0 82952 - 380 0.28080324 12646 0 0 0 79064 - 390 0.30736331 8187 0 0 6250 76480 - 400 0.31631283 16524 0 0 0 58184 - 410 0.33210746 16540 0 0 0 59192 - 420 0.3478275 16195 0 0 0 56848 - 430 0.35397271 15771 0 0 0 53752 - 440 0.36725494 15343 0 0 0 49792 - 450 0.37936057 14912 0 0 0 95040 - 460 0.40185532 14451 0 0 0 90424 - 470 0.41802616 14043 0 0 0 85272 - 480 0.43782941 13632 0 0 0 80800 - 490 0.45607225 9113 0 0 6186 77640 - 500 0.46375317 17020 0 0 0 56560 - 510 0.4767874 17114 0 0 0 58368 - 520 0.49198373 16873 0 0 0 57008 - 530 0.50776837 16381 0 0 0 54128 - 540 0.51378648 15909 0 0 0 50104 - 550 0.52745341 15435 0 0 0 95232 - 560 0.55404357 14945 0 0 0 90248 - 570 0.56638 14487 0 0 0 85608 - 580 0.58150524 14037 0 0 0 81024 - 590 0.59939468 8808 0 0 6129 77592 - 600 0.61722332 17227 0 0 0 55280 - 610 0.62411353 17498 0 0 0 58480 - 620 0.64019107 17196 0 0 0 57504 - 630 0.65584421 16715 0 0 0 54584 - 640 0.67160304 16229 0 0 0 50752 - 650 0.67781786 15763 0 0 0 96552 - 660 0.70218611 15290 0 0 0 91848 - 670 0.71858425 14785 0 0 0 86744 - 680 0.72655947 14302 0 0 0 82312 - 690 0.75199483 9971 0 0 6211 79120 - 700 0.76969197 17883 0 0 0 60192 - 710 0.77678189 17839 0 0 0 60872 - 720 0.79226372 17392 0 0 0 58600 - 730 0.80789016 16903 0 0 0 55008 - 740 0.82394529 16378 0 0 0 50632 - 750 0.83014831 15870 0 0 0 95536 - 760 0.85554017 15402 0 0 0 91176 - 770 0.86387579 14924 0 0 0 86208 - 780 0.88135133 14425 0 0 0 81704 - 790 0.90790929 10148 0 0 6142 78512 - 800 0.91611621 18041 0 0 0 59520 - 810 0.93199114 17883 0 0 0 59480 - 820 0.94813798 17433 0 0 0 57120 - 830 0.96375542 16914 0 0 0 53768 - 840 0.97141773 16401 0 0 0 49840 - 850 0.98354636 15885 0 0 0 94984 - 860 1.0101326 15395 0 0 0 90120 - 870 1.0225388 14887 0 0 0 85384 - 880 1.0372926 14429 0 0 0 80968 - 890 1.0637816 9810 0 0 6167 77696 - 900 1.071539 17450 0 0 0 55136 - 910 1.0879525 17296 0 0 0 54664 - 920 1.0946239 17221 0 0 0 55872 - 930 1.1078292 16851 0 0 0 54040 - 940 1.1235737 16382 0 0 0 50784 - 950 1.1396942 15912 0 0 0 96176 - 960 1.159007 15417 0 0 0 91720 - 970 1.1755201 14903 0 0 0 87184 - 980 1.1835594 14478 0 0 0 83160 - 990 1.2116925 9538 0 0 6156 79936 - 1000 1.2193453 17222 0 0 0 53344 -Loop time of 1.22943 on 1 procs for 1000 steps with 17222 particles - -MPI task timing breakdown: -Section | min time | avg time | max time |%varavg| %total ---------------------------------------------------------------- -Move | 0.76759 | 0.76759 | 0.76759 | 0.0 | 62.43 -Coll | 0 | 0 | 0 | 0.0 | 0.00 -Sort | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.0012273 | 0.0012273 | 0.0012273 | 0.0 | 0.10 -Modify | 0.019763 | 0.019763 | 0.019763 | 0.0 | 1.61 -Output | 0.42493 | 0.42493 | 0.42493 | 0.0 | 34.56 -Other | | 0.01592 | | | 1.30 - -Particle moves = 12312314 (12.3M) -Cells touched = 12415348 (12.4M) -Particle comms = 0 (0K) -Boundary collides = 0 (0K) -Boundary exits = 36524 (36.5K) -SurfColl checks = 54713216 (54.7M) -SurfColl occurs = 56525 (56.5K) -Surf reactions = 56525 (56.5K) -Collide attempts = 0 (0K) -Collide occurs = 0 (0K) -Reactions = 0 (0K) -Particles stuck = 0 -Axisymm bad moves = 0 - -Particle-moves/CPUsec/proc: 1.00146e+07 -Particle-moves/step: 12312.3 -Cell-touches/particle/step: 1.00837 -Particle comm iterations/step: 1 -Particle fraction communicated: 0 -Particle fraction colliding with boundary: 0 -Particle fraction exiting boundary: 0.00296646 -Surface-checks/particle/step: 4.44378 -Surface-collisions/particle/step: 0.00459093 -Surf-reactions/particle/step: 0.00459093 -Collision-attempts/particle/step: 0 -Collisions/particle/step: 0 -Reactions/particle/step: 0 - -Surface reaction tallies: - id adsorb_test_gs_ps2 style adsorb #-of-reactions 14 - reaction all: 104924 - reaction O(g) --> O(s): 42266 - reaction O(g) --> CO(s): 13010 - reaction O(g) --> CO(g): 1246 - reaction C(g) --> C(b): 3 - reaction O(s) --> O(g): 23809 - reaction CO(s) --> CO(g): 14594 - reaction 2O(s) + C(b) --> CO2(g): 7667 - reaction O(s) + C(b) --> CO(s): 2188 - reaction C(b) --> C(g): 141 - -Particles: 17222 ave 17222 max 17222 min -Histogram: 1 0 0 0 0 0 0 0 0 0 -Cells: 8 ave 8 max 8 min -Histogram: 1 0 0 0 0 0 0 0 0 0 -GhostCell: 0 ave 0 max 0 min -Histogram: 1 0 0 0 0 0 0 0 0 0 -EmptyCell: 0 ave 0 max 0 min -Histogram: 1 0 0 0 0 0 0 0 0 0 -Surfs: 12 ave 12 max 12 min -Histogram: 1 0 0 0 0 0 0 0 0 0 -GhostSurf: 0 ave 0 max 0 min -Histogram: 1 0 0 0 0 0 0 0 0 0 diff --git a/examples/surf_react_adsorb/log.11Sep23.mpi_1.beam.surf.ps b/examples/surf_react_adsorb/log.11Sep23.mpi_1.beam.surf.ps deleted file mode 100644 index 355d5771c..000000000 Binary files a/examples/surf_react_adsorb/log.11Sep23.mpi_1.beam.surf.ps and /dev/null differ diff --git a/examples/surf_react_adsorb/log.11Sep23.mpi_4.beam.face.gs b/examples/surf_react_adsorb/log.11Sep23.mpi_4.beam.face.gs deleted file mode 100644 index 558baa0e6..000000000 --- a/examples/surf_react_adsorb/log.11Sep23.mpi_4.beam.face.gs +++ /dev/null @@ -1,238 +0,0 @@ -SPARTA (13 Apr 2023) -Running on 4 MPI task(s) -################################################################################ -# beam of particles striking the surface at an inclined angle -# free molecular flow (no collisions) -# -# Note: -# - The "comm/sort” option to the “global” command is used to match MPI runs. -# The "comm/sort" option should not be used for production runs. -################################################################################ - -seed 123456 -dimension 3 -global gridcut 0.0 comm/sort yes - -boundary oo oo so - - -create_box -11 11 -11 11 0 10 -Created orthogonal box = (-11 -11 0) to (11 11 10) -create_grid 2 2 2 -WARNING: Could not acquire nearby ghost cells b/c grid partition is not clumped (/home/runner/work/sparta/sparta/src/grid.cpp:465) -Created 8 child grid cells - CPU time = 0.0027604 secs - create/ghost percent = 91.1426 8.85741 -balance_grid rcb cell -Balance grid migrated 4 cells - CPU time = 0.0011735 secs - reassign/sort/migrate/ghost percent = 67.2092 0.945888 11.0865 20.7584 - -global nrho 1e10 fnum 1e6 - -species air.species O CO CO2 O2 C -mixture air O O2 vstream 0 1000 -1000 - -mixture air O frac 1.0 -mixture air CO frac 0.0 -mixture air CO2 frac 0.0 -mixture air C frac 0.0 -mixture air O2 frac 0.0 - - -surf_collide 1 cll 300.0 0.5 0.5 0.5 0.5 - -bound_modify zlo collide 1 - -##################################### SURF REACT ADSORB ###################################### -##################################### FACE/BOUNDARY OPTION ################################### - -#surf_react adsorb_test_gs1 adsorb gs sample-GS_1.surf nsync 1 face 1000 6.022e18 O CO -#bound_modify zlo react adsorb_test_gs1 - - -surf_react adsorb_test_gs2 adsorb gs sample-GS_2.surf nsync 1 face 1000 6.022e18 O CO -bound_modify zlo react adsorb_test_gs2 - -########################## BEAM ############################################################ -# Beam at multiple points so that different processors handle the surface collisions - -region circle1 cylinder z 0 -10 1 -INF INF - -fix in1 emit/face/file air zhi data.beam beam_area_1 nevery 100 region circle1 - -################################################################################################ - -#dump 2 image all 10 image.*.ppm type type pdiam 0.2 surf proc 0.01 size 512 512 zoom 1.75 gline no 0.005 -#dump_modify 2 pad 4 - -timestep 0.0001 - -stats 10 -stats_style step cpu np nattempt ncoll nscoll nscheck -run 1000 -Memory usage per proc in Mbytes: - particles (ave,min,max) = 0 0 0 - grid (ave,min,max) = 1.51379 1.51379 1.51379 - surf (ave,min,max) = 0 0 0 - total (ave,min,max) = 1.51379 1.51379 1.51379 -Step CPU Np Natt Ncoll Nscoll Nscheck - 0 0 0 0 0 0 0 - 10 0.0005699 0 0 0 0 0 - 20 0.001200801 0 0 0 0 0 - 30 0.001798901 0 0 0 0 0 - 40 0.002397101 0 0 0 0 0 - 50 0.002973802 0 0 0 0 0 - 60 0.003560302 0 0 0 0 0 - 70 0.004167903 0 0 0 0 0 - 80 0.004787103 0 0 0 0 0 - 90 0.005393503 0 0 0 0 0 - 100 0.009793606 3140 0 0 0 0 - 110 0.010726607 3140 0 0 0 0 - 120 0.011731008 3140 0 0 0 0 - 130 0.012701708 3140 0 0 0 0 - 140 0.013630109 3140 0 0 0 0 - 150 0.014586109 3140 0 0 0 0 - 160 0.01551461 3140 0 0 0 0 - 170 0.016513911 3140 0 0 0 0 - 180 0.017446911 3140 0 0 0 0 - 190 0.020320413 3140 0 0 0 0 - 200 0.022716915 3213 0 0 0 0 - 210 0.023733015 3187 0 0 0 0 - 220 0.024804616 3187 0 0 0 0 - 230 0.026050517 3187 0 0 0 0 - 240 0.027060718 3187 0 0 0 0 - 250 0.028090918 3187 0 0 0 0 - 260 0.029110219 3187 0 0 0 0 - 270 0.03003672 3187 0 0 0 0 - 280 0.03096572 3187 0 0 0 0 - 290 0.031978721 3187 0 0 0 0 - 300 0.035134123 3305 0 0 0 0 - 310 0.036462524 3275 0 0 0 0 - 320 0.037776325 3273 0 0 0 0 - 330 0.039049226 3273 0 0 0 0 - 340 0.040316226 3272 0 0 0 0 - 350 0.041384227 3271 0 0 0 0 - 360 0.042444828 3269 0 0 0 0 - 370 0.043467028 3268 0 0 0 0 - 380 0.044567029 3266 0 0 0 0 - 390 0.04569223 3264 0 0 0 0 - 400 0.049934633 3355 0 0 0 0 - 410 0.051337634 3326 0 0 0 0 - 420 0.052550534 3323 0 0 0 0 - 430 0.053693235 3319 0 0 0 0 - 440 0.054753536 3315 0 0 0 0 - 450 0.055836537 3312 0 0 0 0 - 460 0.056894937 3302 0 0 0 0 - 470 0.057926838 3295 0 0 0 0 - 480 0.058926539 3291 0 0 0 0 - 490 0.060040239 3284 0 0 0 0 - 500 0.063323841 3341 0 0 0 0 - 510 0.064603842 3307 0 0 0 0 - 520 0.065919743 3301 0 0 0 0 - 530 0.067072444 3294 0 0 0 0 - 540 0.068129545 3290 0 0 0 0 - 550 0.069242345 3284 0 0 0 0 - 560 0.070309146 3278 0 0 0 0 - 570 0.071280247 3272 0 0 0 0 - 580 0.072254947 3272 0 0 0 0 - 590 0.073272648 3264 0 0 0 0 - 600 0.07645225 3404 0 0 0 0 - 610 0.077793451 3367 0 0 0 0 - 620 0.079095352 3361 0 0 0 0 - 630 0.080311153 3356 0 0 0 0 - 640 0.081415553 3347 0 0 0 0 - 650 0.082480154 3341 0 0 0 0 - 660 0.083584455 3334 0 0 0 0 - 670 0.084708155 3325 0 0 0 0 - 680 0.085637556 3317 0 0 0 0 - 690 0.086752457 3313 0 0 0 0 - 700 0.089928759 3406 0 0 0 0 - 710 0.09124986 3369 0 0 0 0 - 720 0.092554861 3365 0 0 0 0 - 730 0.093762861 3358 0 0 0 0 - 740 0.094941262 3351 0 0 0 0 - 750 0.096043363 3346 0 0 0 0 - 760 0.096833263 3340 0 0 0 0 - 770 0.097343864 3328 0 0 0 0 - 780 0.097874964 3319 0 0 0 0 - 790 0.098392364 3312 0 0 0 0 - 800 0.099933065 3444 0 0 0 0 - 810 0.10062387 3390 0 0 0 0 - 820 0.10125467 3387 0 0 0 0 - 830 0.10180437 3381 0 0 0 0 - 840 0.10233067 3374 0 0 0 0 - 850 0.10291807 3365 0 0 0 0 - 860 0.10350487 3359 0 0 0 0 - 870 0.10415827 3354 0 0 0 0 - 880 0.10485187 3346 0 0 0 0 - 890 0.10551647 3343 0 0 0 0 - 900 0.10748787 3358 0 0 0 0 - 910 0.10825567 3317 0 0 0 0 - 920 0.10898567 3308 0 0 0 0 - 930 0.10973117 3298 0 0 0 0 - 940 0.11046017 3291 0 0 0 0 - 950 0.11111947 3281 0 0 0 0 - 960 0.11175667 3272 0 0 0 0 - 970 0.11242357 3261 0 0 0 0 - 980 0.11302647 3258 0 0 0 0 - 990 0.11367737 3249 0 0 0 0 - 1000 0.11566528 3393 0 0 0 0 -Loop time of 0.115715 on 4 procs for 1000 steps with 3393 particles - -MPI task timing breakdown: -Section | min time | avg time | max time |%varavg| %total ---------------------------------------------------------------- -Move | 0.0026749 | 0.0093169 | 0.016714 | 6.9 | 8.05 -Coll | 0 | 0 | 0 | 0.0 | 0.00 -Sort | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.034137 | 0.053343 | 0.064234 | 5.2 | 46.10 -Modify | 9.97e-05 | 0.0022262 | 0.0043976 | 4.5 | 1.92 -Output | 0.0052052 | 0.006056 | 0.0072656 | 1.0 | 5.23 -Other | | 0.04477 | | | 38.69 - -Particle moves = 2956516 (2.96M) -Cells touched = 3027028 (3.03M) -Particle comms = 14713 (14.7K) -Boundary collides = 628 (0.628K) -Boundary exits = 401 (0.401K) -SurfColl checks = 0 (0K) -SurfColl occurs = 0 (0K) -Surf reactions = 28231 (28.2K) -Collide attempts = 0 (0K) -Collide occurs = 0 (0K) -Reactions = 0 (0K) -Particles stuck = 0 -Axisymm bad moves = 0 - -Particle-moves/CPUsec/proc: 6.38751e+06 -Particle-moves/step: 2956.52 -Cell-touches/particle/step: 1.02385 -Particle comm iterations/step: 1.42 -Particle fraction communicated: 0.00497647 -Particle fraction colliding with boundary: 0.000212412 -Particle fraction exiting boundary: 0.000135633 -Surface-checks/particle/step: 0 -Surface-collisions/particle/step: 0 -Surf-reactions/particle/step: 0.00954874 -Collision-attempts/particle/step: 0 -Collisions/particle/step: 0 -Reactions/particle/step: 0 - -Surface reaction tallies: - id adsorb_test_gs2 style adsorb #-of-reactions 9 - reaction all: 28231 - reaction O(g) --> O(s): 20944 - reaction O(g) + O(s) --> CO2(g): 4 - reaction O(g) --> CO(s): 6659 - reaction O(g) --> CO(g): 616 - reaction O(g) + O(s) --> O(g) + O(g): 8 - -Particles: 848.25 ave 1642 max 62 min -Histogram: 2 0 0 0 0 0 0 0 0 2 -Cells: 2 ave 2 max 2 min -Histogram: 4 0 0 0 0 0 0 0 0 0 -GhostCell: 6 ave 6 max 6 min -Histogram: 4 0 0 0 0 0 0 0 0 0 -EmptyCell: 6 ave 6 max 6 min -Histogram: 4 0 0 0 0 0 0 0 0 0 diff --git a/examples/surf_react_adsorb/log.11Sep23.mpi_4.beam.face.gs_ps b/examples/surf_react_adsorb/log.11Sep23.mpi_4.beam.face.gs_ps deleted file mode 100644 index d15668b7a..000000000 --- a/examples/surf_react_adsorb/log.11Sep23.mpi_4.beam.face.gs_ps +++ /dev/null @@ -1,241 +0,0 @@ -SPARTA (13 Apr 2023) -Running on 4 MPI task(s) -################################################################################ -# beam of particles striking the surface at an inclined angle -# free molecular flow (no collisions) -# -# Note: -# - The "comm/sort” option to the “global” command is used to match MPI runs. -# The "comm/sort" option should not be used for production runs. -################################################################################ - -seed 123456 -dimension 3 -global gridcut 0.0 comm/sort yes - -boundary oo oo so - - -create_box -11 11 -11 11 0 10 -Created orthogonal box = (-11 -11 0) to (11 11 10) -create_grid 2 2 2 -WARNING: Could not acquire nearby ghost cells b/c grid partition is not clumped (/home/runner/work/sparta/sparta/src/grid.cpp:465) -Created 8 child grid cells - CPU time = 0.001817 secs - create/ghost percent = 92.0914 7.90864 -balance_grid rcb cell -Balance grid migrated 4 cells - CPU time = 0.000641 secs - reassign/sort/migrate/ghost percent = 67.3791 0.826833 10.858 20.936 - -global nrho 1e10 fnum 1e6 - -species air.species O CO CO2 O2 C -mixture air O O2 vstream 0 1000 -1000 - -mixture air O frac 1.0 -mixture air CO frac 0.0 -mixture air CO2 frac 0.0 -mixture air C frac 0.0 -mixture air O2 frac 0.0 - - -surf_collide 1 cll 300.0 0.5 0.5 0.5 0.5 - -bound_modify zlo collide 1 - -##################################### SURF REACT ADSORB ###################################### -##################################### FACE/BOUNDARY OPTION ################################### - -#surf_react adsorb_test_gs_ps1 adsorb gs/ps sample-GS_1.surf sample-PS_1.surf nsync 1 face 1000 6.022e18 O CO -#bound_modify zlo react adsorb_test_gs_ps1 - - -surf_react adsorb_test_gs_ps2 adsorb gs/ps sample-GS_2.surf sample-PS_2.surf nsync 1 face 1000 6.022e18 O CO -bound_modify zlo react adsorb_test_gs_ps2 - -########################## BEAM ############################################################ -# Beam at multiple points so that different processors handle the surface collisions - -region circle1 cylinder z 0 -10 1 -INF INF - -fix in1 emit/face/file air zhi data.beam beam_area_1 nevery 100 region circle1 - -################################################################################################ - -#dump 2 image all 10 image.*.ppm type type pdiam 0.2 surf proc 0.01 size 512 512 zoom 1.75 gline no 0.005 -#dump_modify 2 pad 4 - -timestep 0.0001 - -stats 10 -stats_style step cpu np nattempt ncoll nscoll nscheck -run 1000 -Memory usage per proc in Mbytes: - particles (ave,min,max) = 0 0 0 - grid (ave,min,max) = 1.51379 1.51379 1.51379 - surf (ave,min,max) = 0 0 0 - total (ave,min,max) = 1.51379 1.51379 1.51379 -Step CPU Np Natt Ncoll Nscoll Nscheck - 0 0 0 0 0 0 0 - 10 0.000409 0 0 0 0 0 - 20 0.0008664 0 0 0 0 0 - 30 0.001278801 0 0 0 0 0 - 40 0.001689701 0 0 0 0 0 - 50 0.002135101 0 0 0 0 0 - 60 0.002552802 0 0 0 0 0 - 70 0.002969902 0 0 0 0 0 - 80 0.003382802 0 0 0 0 0 - 90 0.003842602 0 0 0 0 0 - 100 0.005599304 3116 0 0 0 0 - 110 0.007142005 3117 0 0 0 0 - 120 0.007741705 3117 0 0 0 0 - 130 0.008324205 3117 0 0 0 0 - 140 0.008887906 3117 0 0 0 0 - 150 0.009450806 3117 0 0 0 0 - 160 0.010067507 3117 0 0 0 0 - 170 0.010644007 3117 0 0 0 0 - 180 0.012292308 3118 0 0 0 0 - 190 0.012915208 3120 0 0 0 0 - 200 0.01562851 3785 0 0 0 0 - 210 0.018344912 5769 0 0 0 0 - 220 0.019755013 6033 0 0 0 0 - 230 0.020952114 6041 0 0 0 0 - 240 0.022036714 5978 0 0 0 0 - 250 0.023099915 5929 0 0 0 0 - 260 0.024137016 5866 0 0 0 0 - 270 0.025162416 5798 0 0 0 0 - 280 0.026200417 5718 0 0 0 0 - 290 0.027225918 5630 0 0 0 0 - 300 0.03029752 6457 0 0 0 0 - 310 0.032814821 7925 0 0 0 0 - 320 0.034316722 8037 0 0 0 0 - 330 0.035581923 7896 0 0 0 0 - 340 0.036754024 7706 0 0 0 0 - 350 0.038002625 7502 0 0 0 0 - 360 0.039125026 7283 0 0 0 0 - 370 0.040235526 7072 0 0 0 0 - 380 0.041337527 6872 0 0 0 0 - 390 0.042420728 6671 0 0 0 0 - 400 0.04526073 7272 0 0 0 0 - 410 0.047942231 8759 0 0 0 0 - 420 0.049445432 8738 0 0 0 0 - 430 0.050776333 8570 0 0 0 0 - 440 0.052002834 8311 0 0 0 0 - 450 0.053238335 8067 0 0 0 0 - 460 0.054411736 7822 0 0 0 0 - 470 0.055550436 7605 0 0 0 0 - 480 0.056653937 7394 0 0 0 0 - 490 0.057761338 7139 0 0 0 0 - 500 0.06072614 7879 0 0 0 0 - 510 0.063369041 9279 0 0 0 0 - 520 0.064870142 9197 0 0 0 0 - 530 0.066178443 8959 0 0 0 0 - 540 0.067416044 8672 0 0 0 0 - 550 0.068651345 8391 0 0 0 0 - 560 0.069815146 8151 0 0 0 0 - 570 0.071002946 7882 0 0 0 0 - 580 0.072116847 7632 0 0 0 0 - 590 0.073212548 7358 0 0 0 0 - 600 0.07624405 8067 0 0 0 0 - 610 0.078804952 9368 0 0 0 0 - 620 0.080316753 9304 0 0 0 0 - 630 0.081625953 9101 0 0 0 0 - 640 0.082877254 8831 0 0 0 0 - 650 0.084140455 8545 0 0 0 0 - 660 0.085307956 8275 0 0 0 0 - 670 0.086684157 7995 0 0 0 0 - 680 0.087830358 7744 0 0 0 0 - 690 0.089006658 7485 0 0 0 0 - 700 0.09161356 7685 0 0 0 0 - 710 0.094133262 8923 0 0 0 0 - 720 0.095863563 9143 0 0 0 0 - 730 0.097341364 9072 0 0 0 0 - 740 0.098747865 8866 0 0 0 0 - 750 0.10003507 8588 0 0 0 0 - 760 0.10123167 8291 0 0 0 0 - 770 0.10241887 8029 0 0 0 0 - 780 0.10357257 7793 0 0 0 0 - 790 0.10467417 7541 0 0 0 0 - 800 0.10723097 7684 0 0 0 0 - 810 0.10896807 7931 0 0 0 0 - 820 0.11144827 9138 0 0 0 0 - 830 0.11306727 9198 0 0 0 0 - 840 0.11439718 8950 0 0 0 0 - 850 0.11568118 8678 0 0 0 0 - 860 0.11687888 8406 0 0 0 0 - 870 0.11806348 8157 0 0 0 0 - 880 0.11921238 7916 0 0 0 0 - 890 0.12034408 7659 0 0 0 0 - 900 0.12335038 8376 0 0 0 0 - 910 0.12601328 9720 0 0 0 0 - 920 0.12752838 9685 0 0 0 0 - 930 0.12893598 9488 0 0 0 0 - 940 0.13025398 9207 0 0 0 0 - 950 0.13155519 8914 0 0 0 0 - 960 0.13275919 8619 0 0 0 0 - 970 0.13391929 8331 0 0 0 0 - 980 0.13511859 8037 0 0 0 0 - 990 0.13625609 7775 0 0 0 0 - 1000 0.13917909 8331 0 0 0 0 -Loop time of 0.139245 on 4 procs for 1000 steps with 8331 particles - -MPI task timing breakdown: -Section | min time | avg time | max time |%varavg| %total ---------------------------------------------------------------- -Move | 0.011289 | 0.017858 | 0.024592 | 4.9 | 12.83 -Coll | 0 | 0 | 0 | 0.0 | 0.00 -Sort | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.028203 | 0.035742 | 0.044103 | 3.6 | 25.67 -Modify | 7.9e-05 | 0.0022028 | 0.004335 | 4.5 | 1.58 -Output | 0.011373 | 0.012553 | 0.014586 | 1.1 | 9.02 -Other | | 0.07089 | | | 50.91 - -Particle moves = 6647023 (6.65M) -Cells touched = 6758950 (6.76M) -Particle comms = 24016 (24K) -Boundary collides = 638 (0.638K) -Boundary exits = 18948 (18.9K) -SurfColl checks = 0 (0K) -SurfColl occurs = 0 (0K) -Surf reactions = 28340 (28.3K) -Collide attempts = 0 (0K) -Collide occurs = 0 (0K) -Reactions = 0 (0K) -Particles stuck = 0 -Axisymm bad moves = 0 - -Particle-moves/CPUsec/proc: 1.1934e+07 -Particle-moves/step: 6647.02 -Cell-touches/particle/step: 1.01684 -Particle comm iterations/step: 1.822 -Particle fraction communicated: 0.00361305 -Particle fraction colliding with boundary: 9.59828e-05 -Particle fraction exiting boundary: 0.0028506 -Surface-checks/particle/step: 0 -Surface-collisions/particle/step: 0 -Surf-reactions/particle/step: 0.00426356 -Collision-attempts/particle/step: 0 -Collisions/particle/step: 0 -Reactions/particle/step: 0 - -Surface reaction tallies: - id adsorb_test_gs_ps2 style adsorb #-of-reactions 14 - reaction all: 52825 - reaction O(g) --> O(s): 21202 - reaction O(g) --> CO(s): 6500 - reaction O(g) --> CO(g): 638 - reaction O(s) --> O(g): 14614 - reaction CO(s) --> CO(g): 6669 - reaction 2O(s) + C(b) --> CO2(g): 2203 - reaction O(s) + C(b) --> CO(s): 987 - reaction C(b) --> C(g): 12 - -Particles: 2082.75 ave 2904 max 1256 min -Histogram: 2 0 0 0 0 0 0 0 0 2 -Cells: 2 ave 2 max 2 min -Histogram: 4 0 0 0 0 0 0 0 0 0 -GhostCell: 6 ave 6 max 6 min -Histogram: 4 0 0 0 0 0 0 0 0 0 -EmptyCell: 6 ave 6 max 6 min -Histogram: 4 0 0 0 0 0 0 0 0 0 diff --git a/examples/surf_react_adsorb/log.11Sep23.mpi_4.beam.face.ps b/examples/surf_react_adsorb/log.11Sep23.mpi_4.beam.face.ps deleted file mode 100644 index 6a80cef53..000000000 Binary files a/examples/surf_react_adsorb/log.11Sep23.mpi_4.beam.face.ps and /dev/null differ diff --git a/examples/surf_react_adsorb/log.11Sep23.mpi_4.beam.surf.gs b/examples/surf_react_adsorb/log.11Sep23.mpi_4.beam.surf.gs deleted file mode 100644 index a5b4b36ac..000000000 --- a/examples/surf_react_adsorb/log.11Sep23.mpi_4.beam.surf.gs +++ /dev/null @@ -1,258 +0,0 @@ -SPARTA (13 Apr 2023) -Running on 4 MPI task(s) -################################################################################ -# beam of particles striking the surface at an inclined angle -# free molecular flow (no collisions) -# -# Note: -# - The "comm/sort” option to the “global” command is used to match MPI runs. -# The "comm/sort" option should not be used for production runs. -################################################################################ - -seed 123456 -dimension 3 -global gridcut 0.0 comm/sort yes - -boundary oo oo oo - - -create_box -11 11 -11 11 0 10 -Created orthogonal box = (-11 -11 0) to (11 11 10) -create_grid 2 2 2 -WARNING: Could not acquire nearby ghost cells b/c grid partition is not clumped (/home/runner/work/sparta/sparta/src/grid.cpp:465) -Created 8 child grid cells - CPU time = 0.0018162 secs - create/ghost percent = 92.3302 7.66985 -balance_grid rcb cell -Balance grid migrated 4 cells - CPU time = 0.000645301 secs - reassign/sort/migrate/ghost percent = 66.6202 0.821322 11.359 21.1994 - -global nrho 1e10 fnum 1e6 - -species air.species O CO CO2 O2 C -mixture air O O2 vstream 0 1000 -1000 - -mixture air O frac 1.0 -mixture air CO frac 0.0 -mixture air CO2 frac 0.0 -mixture air C frac 0.0 -mixture air O2 frac 0.0 - - -surf_collide 1 cll 300.0 0.5 0.5 0.5 0.5 - -read_surf base_plate.surf - 8 points - 12 triangles - -11 11 xlo xhi - -11 11 ylo yhi - 0 1 zlo zhi - 1 min triangle edge length - 11 min triangle area - 4 0 = cells overlapping surfs, overlap cells with unmarked corner pts - 4 0 4 = cells outside/inside/overlapping surfs - 4 = surf cells with 1,2,etc splits - 4356 4356 = cell-wise and global flow volume - CPU time = 0.0010869 secs - read/check/sort/surf2grid/ghost/inout/particle percent = 21.0875 9.10855 0.60723 55.5249 13.6719 11.7306 1.00285 - surf2grid time = 0.000603501 secs - map/comm1/comm2/comm3/comm4/split percent = 32.1292 10.3562 5.23611 5.36867 16.5534 23.8442 - -##################################### SURF REACT ADSORB ###################################### -##################################### SURF OPTION ############################################ - -#surf_react adsorb_test_gs1 adsorb gs sample-GS_1.surf nsync 1 surf 1000 6.022e18 O CO -#surf_modify all collide 1 react adsorb_test_gs1 - -surf_react adsorb_test_gs2 adsorb gs sample-GS_2.surf nsync 1 surf 1000 6.022e18 O CO -surf_modify all collide 1 react adsorb_test_gs2 - -########################## BEAM ############################################################ -# Beam at multiple points so that different processors handle the surface collisions - -region circle2 cylinder z 6 -10 1 -INF INF -region circle3 cylinder z -6 -10 1 -INF INF - -fix in2 emit/face/file air zhi data.beam beam_area_2 nevery 100 region circle2 -fix in3 emit/face/file air zhi data.beam beam_area_3 nevery 100 region circle3 - -################################################################################################ - -#dump 2 image all 10 image.*.ppm type type pdiam 0.2 surf proc 0.01 size 512 512 zoom 1.75 gline no 0.005 -#dump_modify 2 pad 4 - -timestep 0.0001 - -stats 10 -stats_style step cpu np nattempt ncoll nscoll nscheck -run 1000 -Memory usage per proc in Mbytes: - particles (ave,min,max) = 0 0 0 - grid (ave,min,max) = 1.51379 1.51379 1.51379 - surf (ave,min,max) = 0.00151062 0.00151062 0.00151062 - total (ave,min,max) = 1.5153 1.5153 1.5153 -Step CPU Np Natt Ncoll Nscoll Nscheck - 0 0 0 0 0 0 0 - 10 0.000435501 0 0 0 0 0 - 20 0.000858401 0 0 0 0 0 - 30 0.001286002 0 0 0 0 0 - 40 0.001713803 0 0 0 0 0 - 50 0.002131903 0 0 0 0 0 - 60 0.002576704 0 0 0 0 0 - 70 0.003013904 0 0 0 0 0 - 80 0.003432605 0 0 0 0 0 - 90 0.003847506 0 0 0 0 0 - 100 0.007605011 6273 0 0 0 0 - 110 0.008798413 6273 0 0 0 0 - 120 0.009956914 6273 0 0 0 0 - 130 0.011152016 6273 0 0 0 0 - 140 0.012304518 6273 0 0 0 0 - 150 0.01385412 6273 0 0 0 49952 - 160 0.018253626 6273 0 0 0 50184 - 170 0.022685933 6273 0 0 0 50184 - 180 0.027090439 6273 0 0 0 50184 - 190 0.034282649 182 0 0 6178 50888 - 200 0.037477854 6450 0 0 0 1040 - 210 0.038834056 6450 0 0 0 1016 - 220 0.040243558 6450 0 0 0 1016 - 230 0.04156996 6450 0 0 0 1016 - 240 0.042885661 6450 0 0 0 1000 - 250 0.044558064 6450 0 0 0 51336 - 260 0.04910687 6450 0 0 0 51432 - 270 0.053632877 6449 0 0 0 51336 - 280 0.058159183 6448 0 0 0 51248 - 290 0.064553192 310 0 0 6233 51928 - 300 0.066657695 6572 0 0 0 1656 - 310 0.068065997 6569 0 0 0 1608 - 320 0.069483699 6564 0 0 0 1544 - 330 0.070940802 6556 0 0 0 1480 - 340 0.072354904 6551 0 0 0 1424 - 350 0.074111006 6545 0 0 0 51608 - 360 0.078767413 6537 0 0 0 51688 - 370 0.083373019 6529 0 0 0 51536 - 380 0.088165126 6520 0 0 0 51352 - 390 0.094565135 414 0 0 6211 52152 - 400 0.096708938 6616 0 0 0 1976 - 410 0.098188441 6604 0 0 0 1880 - 420 0.099665543 6598 0 0 0 1792 - 430 0.10103894 6587 0 0 0 1776 - 440 0.10243875 6572 0 0 0 1736 - 450 0.10419845 6563 0 0 0 51400 - 460 0.10881336 6555 0 0 0 51496 - 470 0.11345416 6541 0 0 0 51328 - 480 0.11797257 6533 0 0 0 51224 - 490 0.12433728 454 0 0 6141 51800 - 500 0.12654568 6662 0 0 0 2064 - 510 0.12802228 6643 0 0 0 1968 - 520 0.12950619 6632 0 0 0 1888 - 530 0.13124809 6622 0 0 0 1824 - 540 0.13341009 6607 0 0 0 1784 - 550 0.13531119 6588 0 0 0 51632 - 560 0.1399347 6575 0 0 0 51736 - 570 0.14461811 6555 0 0 0 51552 - 580 0.14921931 6546 0 0 0 51448 - 590 0.15568462 467 0 0 6171 52136 - 600 0.15781183 6684 0 0 0 2208 - 610 0.15926943 6675 0 0 0 2032 - 620 0.16075813 6659 0 0 0 1920 - 630 0.16219033 6645 0 0 0 1840 - 640 0.16355783 6628 0 0 0 1720 - 650 0.16539634 6615 0 0 0 51688 - 660 0.17005184 6602 0 0 0 51704 - 670 0.17481635 6587 0 0 0 51536 - 680 0.17944476 6572 0 0 0 51312 - 690 0.18587147 480 0 0 6196 52064 - 700 0.18800467 6714 0 0 0 2064 - 710 0.18952947 6700 0 0 0 1976 - 720 0.19102727 6686 0 0 0 1888 - 730 0.19244518 6673 0 0 0 1840 - 740 0.19391038 6650 0 0 0 1696 - 750 0.19578418 6630 0 0 0 51664 - 760 0.20043169 6620 0 0 0 51744 - 770 0.20508859 6601 0 0 0 51600 - 780 0.2097361 6583 0 0 0 51440 - 790 0.21616761 454 0 0 6205 51992 - 800 0.21832481 6684 0 0 0 1968 - 810 0.21978062 6661 0 0 0 1888 - 820 0.22128052 6640 0 0 0 1768 - 830 0.22271612 6624 0 0 0 1696 - 840 0.22410482 6615 0 0 0 1600 - 850 0.22586512 6606 0 0 0 51504 - 860 0.23049193 6590 0 0 0 51600 - 870 0.23514114 6573 0 0 0 51496 - 880 0.23971404 6558 0 0 0 51352 - 890 0.24607915 478 0 0 6183 52152 - 900 0.24818235 6581 0 0 0 2160 - 910 0.24966116 6562 0 0 0 2088 - 920 0.25110526 6550 0 0 0 2032 - 930 0.25250516 6535 0 0 0 1904 - 940 0.25391626 6510 0 0 0 1736 - 950 0.25575137 6496 0 0 0 50656 - 960 0.26031517 6485 0 0 0 50760 - 970 0.26484298 6472 0 0 0 50600 - 980 0.26935568 6459 0 0 0 50480 - 990 0.27567349 476 0 0 6060 51096 - 1000 0.2777582 6645 0 0 0 2040 -Loop time of 0.277808 on 4 procs for 1000 steps with 6645 particles - -MPI task timing breakdown: -Section | min time | avg time | max time |%varavg| %total ---------------------------------------------------------------- -Move | 0.0020757 | 0.052934 | 0.10429 | 22.1 | 19.05 -Coll | 0 | 0 | 0 | 0.0 | 0.00 -Sort | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.020101 | 0.022731 | 0.026772 | 1.6 | 8.18 -Modify | 8.69e-05 | 0.0038357 | 0.0075932 | 6.1 | 1.38 -Output | 0.0049898 | 0.0077064 | 0.015178 | 4.9 | 2.77 -Other | | 0.1906 | | | 68.61 - -Particle moves = 5369855 (5.37M) -Cells touched = 5447071 (5.45M) -Particle comms = 725 (0.725K) -Boundary collides = 0 (0K) -Boundary exits = 941 (0.941K) -SurfColl checks = 19760352 (19.8M) -SurfColl occurs = 56465 (56.5K) -Surf reactions = 56465 (56.5K) -Collide attempts = 0 (0K) -Collide occurs = 0 (0K) -Reactions = 0 (0K) -Particles stuck = 0 -Axisymm bad moves = 0 - -Particle-moves/CPUsec/proc: 4.83235e+06 -Particle-moves/step: 5369.85 -Cell-touches/particle/step: 1.01438 -Particle comm iterations/step: 1.456 -Particle fraction communicated: 0.000135013 -Particle fraction colliding with boundary: 0 -Particle fraction exiting boundary: 0.000175238 -Surface-checks/particle/step: 3.67987 -Surface-collisions/particle/step: 0.0105152 -Surf-reactions/particle/step: 0.0105152 -Collision-attempts/particle/step: 0 -Collisions/particle/step: 0 -Reactions/particle/step: 0 - -Surface reaction tallies: - id adsorb_test_gs2 style adsorb #-of-reactions 9 - reaction all: 56465 - reaction O(g) --> O(s): 41989 - reaction O(g) + O(s) --> CO2(g): 9 - reaction O(g) --> CO(s): 13164 - reaction O(g) --> CO(g): 1275 - reaction O(g) + O(s) --> O(g) + O(g): 28 - -Particles: 1661.25 ave 3277 max 54 min -Histogram: 2 0 0 0 0 0 0 0 0 2 -Cells: 2 ave 2 max 2 min -Histogram: 4 0 0 0 0 0 0 0 0 0 -GhostCell: 6 ave 6 max 6 min -Histogram: 4 0 0 0 0 0 0 0 0 0 -EmptyCell: 6 ave 6 max 6 min -Histogram: 4 0 0 0 0 0 0 0 0 0 -Surfs: 12 ave 12 max 12 min -Histogram: 4 0 0 0 0 0 0 0 0 0 -GhostSurf: 0 ave 0 max 0 min -Histogram: 4 0 0 0 0 0 0 0 0 0 diff --git a/examples/surf_react_adsorb/log.11Sep23.mpi_4.beam.surf.gs_ps b/examples/surf_react_adsorb/log.11Sep23.mpi_4.beam.surf.gs_ps deleted file mode 100644 index c94722388..000000000 --- a/examples/surf_react_adsorb/log.11Sep23.mpi_4.beam.surf.gs_ps +++ /dev/null @@ -1,262 +0,0 @@ -SPARTA (13 Apr 2023) -Running on 4 MPI task(s) -################################################################################ -# beam of particles striking the surface at an inclined angle -# free molecular flow (no collisions) -# -# Note: -# - The "comm/sort” option to the “global” command is used to match MPI runs. -# The "comm/sort" option should not be used for production runs. -################################################################################ - -seed 123456 -dimension 3 -global gridcut 0.0 comm/sort yes - -boundary oo oo oo - - -create_box -11 11 -11 11 0 10 -Created orthogonal box = (-11 -11 0) to (11 11 10) -create_grid 2 2 2 -WARNING: Could not acquire nearby ghost cells b/c grid partition is not clumped (/home/runner/work/sparta/sparta/src/grid.cpp:465) -Created 8 child grid cells - CPU time = 0.0030228 secs - create/ghost percent = 85.2058 14.7942 -balance_grid rcb cell -Balance grid migrated 4 cells - CPU time = 0.0011449 secs - reassign/sort/migrate/ghost percent = 66.6347 1.05686 11.18 21.1284 - -global nrho 1e10 fnum 1e6 - -species air.species O CO CO2 O2 C -mixture air O O2 vstream 0 1000 -1000 - -mixture air O frac 1.0 -mixture air CO frac 0.0 -mixture air CO2 frac 0.0 -mixture air C frac 0.0 -mixture air O2 frac 0.0 - - -surf_collide 1 cll 300.0 0.5 0.5 0.5 0.5 - -read_surf base_plate.surf - 8 points - 12 triangles - -11 11 xlo xhi - -11 11 ylo yhi - 0 1 zlo zhi - 1 min triangle edge length - 11 min triangle area - 4 0 = cells overlapping surfs, overlap cells with unmarked corner pts - 4 0 4 = cells outside/inside/overlapping surfs - 4 = surf cells with 1,2,etc splits - 4356 4356 = cell-wise and global flow volume - CPU time = 0.0016718 secs - read/check/sort/surf2grid/ghost/inout/particle percent = 19.4162 9.80977 1.19033 53.4274 16.1563 12.7527 0.783585 - surf2grid time = 0.000893201 secs - map/comm1/comm2/comm3/comm4/split percent = 27.8549 12.7071 6.87426 6.70622 11.8562 24.3842 - -##################################### SURF REACT ADSORB ###################################### -##################################### SURF OPTION ############################################ - -#surf_react adsorb_test_gs_ps1 adsorb gs/ps sample-GS_1.surf sample-PS_1.surf nsync 1 surf 1000 6.022e18 O CO -#surf_modify all collide 1 react adsorb_test_gs_ps1 - -surf_react adsorb_test_gs_ps2 adsorb gs/ps sample-GS_2.surf sample-PS_2.surf nsync 1 surf 1000 6.022e18 O CO -surf_modify all collide 1 react adsorb_test_gs_ps2 - -########################## BEAM ############################################################ -# Beam at multiple points so that different processors handle the surface collisions - -region circle2 cylinder z 6 -10 1 -INF INF -region circle3 cylinder z -6 -10 1 -INF INF - -fix in2 emit/face/file air zhi data.beam beam_area_2 nevery 100 region circle2 -fix in3 emit/face/file air zhi data.beam beam_area_3 nevery 100 region circle3 - -################################################################################################ - -#dump 2 image all 10 image.*.ppm type type pdiam 0.2 surf proc 0.01 size 512 512 zoom 1.75 gline no 0.005 -#dump_modify 2 pad 4 - -timestep 0.0001 - -stats 10 -stats_style step cpu np nattempt ncoll nscoll nscheck -run 1000 -Memory usage per proc in Mbytes: - particles (ave,min,max) = 0 0 0 - grid (ave,min,max) = 1.51379 1.51379 1.51379 - surf (ave,min,max) = 0.00151062 0.00151062 0.00151062 - total (ave,min,max) = 1.5153 1.5153 1.5153 -Step CPU Np Natt Ncoll Nscoll Nscheck - 0 0 0 0 0 0 0 - 10 0.001519902 0 0 0 0 0 - 20 0.003020404 0 0 0 0 0 - 30 0.004547707 0 0 0 0 0 - 40 0.006178209 0 0 0 0 0 - 50 0.007863911 0 0 0 0 0 - 60 0.009520814 0 0 0 0 0 - 70 0.011216316 0 0 0 0 0 - 80 0.012873419 0 0 0 0 0 - 90 0.014526621 0 0 0 0 0 - 100 0.02112173 6302 0 0 0 0 - 110 0.025204736 6304 0 0 0 16 - 120 0.027245439 6304 0 0 0 16 - 130 0.029986443 6306 0 0 0 24 - 140 0.032954647 6305 0 0 0 24 - 150 0.036268352 6305 0 0 0 50224 - 160 0.043844263 6305 0 0 0 50448 - 170 0.051431874 6306 0 0 0 50440 - 180 0.059063685 6306 0 0 0 50440 - 190 0.070501201 1698 0 0 6211 51240 - 200 0.075530708 11613 0 0 0 41376 - 210 0.079399714 12044 0 0 0 45736 - 220 0.083295019 12024 0 0 0 45792 - 230 0.086990525 11909 0 0 0 44152 - 240 0.09059473 11790 0 0 0 40968 - 250 0.094494635 11677 0 0 0 87240 - 260 0.10121115 11529 0 0 0 82624 - 270 0.10771375 11326 0 0 0 77904 - 280 0.11408666 11128 0 0 0 73920 - 290 0.12358148 6998 0 0 6231 71312 - 300 0.12964749 14673 0 0 0 48616 - 310 0.13463859 15093 0 0 0 51840 - 320 0.1392557 14972 0 0 0 51152 - 330 0.14371711 14651 0 0 0 48712 - 340 0.14800521 14295 0 0 0 45544 - 350 0.15247932 13935 0 0 0 91920 - 360 0.15959103 13561 0 0 0 87800 - 370 0.16648254 13202 0 0 0 83656 - 380 0.17320915 12802 0 0 0 79224 - 390 0.18226066 7362 0 0 6252 76192 - 400 0.18820297 15409 0 0 0 46440 - 410 0.19365048 16390 0 0 0 56216 - 420 0.19852998 16257 0 0 0 56008 - 430 0.20319439 15923 0 0 0 53528 - 440 0.2076509 15481 0 0 0 50376 - 450 0.2122767 15045 0 0 0 96456 - 460 0.21963561 14607 0 0 0 91680 - 470 0.22676212 14173 0 0 0 87072 - 480 0.23373914 13750 0 0 0 82640 - 490 0.24373115 9460 0 0 6243 79648 - 500 0.24991246 16768 0 0 0 55400 - 510 0.25511577 16971 0 0 0 57456 - 520 0.26009697 16716 0 0 0 56144 - 530 0.26485168 16284 0 0 0 53296 - 540 0.26940489 15837 0 0 0 49560 - 550 0.27412189 15348 0 0 0 94792 - 560 0.2814846 14887 0 0 0 90096 - 570 0.28864141 14465 0 0 0 85776 - 580 0.29555722 14027 0 0 0 81264 - 590 0.30517234 9712 0 0 6150 78472 - 600 0.31174005 17629 0 0 0 58560 - 610 0.31707465 17630 0 0 0 59744 - 620 0.32214256 17278 0 0 0 58112 - 630 0.32701017 16819 0 0 0 55072 - 640 0.33170017 16312 0 0 0 51208 - 650 0.33661488 15823 0 0 0 96624 - 660 0.34410279 15297 0 0 0 91720 - 670 0.3513365 14832 0 0 0 87088 - 680 0.35838091 14342 0 0 0 82408 - 690 0.36774653 9329 0 0 6218 78880 - 700 0.37364453 16746 0 0 0 50256 - 710 0.37876884 17142 0 0 0 53464 - 720 0.38414505 17407 0 0 0 57624 - 730 0.38894436 16958 0 0 0 55048 - 740 0.39362736 16472 0 0 0 51736 - 750 0.39849157 15982 0 0 0 97792 - 760 0.40609888 15506 0 0 0 92944 - 770 0.41391709 15021 0 0 0 88200 - 780 0.4210808 14553 0 0 0 83376 - 790 0.43114342 10095 0 0 6189 80168 - 800 0.43802413 17897 0 0 0 58576 - 810 0.44353114 18130 0 0 0 61280 - 820 0.44879094 17830 0 0 0 59848 - 830 0.45376765 17353 0 0 0 56616 - 840 0.45861126 16812 0 0 0 52424 - 850 0.46365586 16312 0 0 0 97504 - 860 0.47121947 15850 0 0 0 92800 - 870 0.47855668 15303 0 0 0 87688 - 880 0.48571879 14778 0 0 0 82776 - 890 0.49528881 10001 0 0 6197 79432 - 900 0.50191782 17987 0 0 0 57168 - 910 0.50739043 18082 0 0 0 60040 - 920 0.51249223 17608 0 0 0 57608 - 930 0.51741134 17101 0 0 0 54712 - 940 0.52211465 16579 0 0 0 51000 - 950 0.52707765 16111 0 0 0 96568 - 960 0.53456857 15624 0 0 0 92104 - 970 0.54188888 15136 0 0 0 87544 - 980 0.54899119 14668 0 0 0 82992 - 990 0.5590333 10630 0 0 6154 79904 - 1000 0.56592491 18480 0 0 0 62512 -Loop time of 0.566008 on 4 procs for 1000 steps with 18480 particles - -MPI task timing breakdown: -Section | min time | avg time | max time |%varavg| %total ---------------------------------------------------------------- -Move | 0.088184 | 0.14236 | 0.20034 | 14.2 | 25.15 -Coll | 0 | 0 | 0 | 0.0 | 0.00 -Sort | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.054657 | 0.11348 | 0.15609 | 13.0 | 20.05 -Modify | 0.000114 | 0.0038966 | 0.0077387 | 6.1 | 0.69 -Output | 0.0258 | 0.027056 | 0.029761 | 1.0 | 4.78 -Other | | 0.2792 | | | 49.33 - -Particle moves = 12310830 (12.3M) -Cells touched = 12499733 (12.5M) -Particle comms = 17703 (17.7K) -Boundary collides = 0 (0K) -Boundary exits = 36205 (36.2K) -SurfColl checks = 55117792 (55.1M) -SurfColl occurs = 56763 (56.8K) -Surf reactions = 56762 (56.8K) -Collide attempts = 0 (0K) -Collide occurs = 0 (0K) -Reactions = 0 (0K) -Particles stuck = 0 -Axisymm bad moves = 0 - -Particle-moves/CPUsec/proc: 5.43757e+06 -Particle-moves/step: 12310.8 -Cell-touches/particle/step: 1.01534 -Particle comm iterations/step: 1.825 -Particle fraction communicated: 0.001438 -Particle fraction colliding with boundary: 0 -Particle fraction exiting boundary: 0.00294091 -Surface-checks/particle/step: 4.47718 -Surface-collisions/particle/step: 0.00461082 -Surf-reactions/particle/step: 0.00461074 -Collision-attempts/particle/step: 0 -Collisions/particle/step: 0 -Reactions/particle/step: 0 - -Surface reaction tallies: - id adsorb_test_gs_ps2 style adsorb #-of-reactions 14 - reaction all: 106047 - reaction O(g) --> O(s): 42321 - reaction O(g) --> CO(s): 13196 - reaction O(g) --> CO(g): 1242 - reaction C(g) --> C(b): 3 - reaction O(s) --> O(g): 24122 - reaction CO(s) --> CO(g): 14914 - reaction 2O(s) + C(b) --> CO2(g): 7979 - reaction O(s) + C(b) --> CO(s): 2140 - reaction C(b) --> C(g): 130 - -Particles: 4620 ave 6365 max 2951 min -Histogram: 2 0 0 0 0 0 0 0 0 2 -Cells: 2 ave 2 max 2 min -Histogram: 4 0 0 0 0 0 0 0 0 0 -GhostCell: 6 ave 6 max 6 min -Histogram: 4 0 0 0 0 0 0 0 0 0 -EmptyCell: 6 ave 6 max 6 min -Histogram: 4 0 0 0 0 0 0 0 0 0 -Surfs: 12 ave 12 max 12 min -Histogram: 4 0 0 0 0 0 0 0 0 0 -GhostSurf: 0 ave 0 max 0 min -Histogram: 4 0 0 0 0 0 0 0 0 0 diff --git a/examples/surf_react_adsorb/log.11Sep23.mpi_4.beam.surf.ps b/examples/surf_react_adsorb/log.11Sep23.mpi_4.beam.surf.ps deleted file mode 100644 index 09009df4c..000000000 Binary files a/examples/surf_react_adsorb/log.11Sep23.mpi_4.beam.surf.ps and /dev/null differ diff --git a/examples/surf_react_adsorb/log.22Aug26.mpi_1.beam.face.gs b/examples/surf_react_adsorb/log.22Aug26.mpi_1.beam.face.gs new file mode 100644 index 000000000..6aef18157 --- /dev/null +++ b/examples/surf_react_adsorb/log.22Aug26.mpi_1.beam.face.gs @@ -0,0 +1,240 @@ +SPARTA (24 Sep 2025) +Running on 1 MPI task(s) +################################################################################ +# beam of particles striking the surface at an inclined angle +# free molecular flow (no collisions) +# +# Note: +# - The "comm/sort” option to the “global” command is used to match MPI runs. +# The "comm/sort" option should not be used for production runs. +################################################################################ + +seed 123456 +dimension 3 +global gridcut 0.0 comm/sort yes + +boundary oo oo so + + +create_box -11 11 -11 11 0 10 +Created orthogonal box = (-11 -11 0) to (11 11 10) +create_grid 2 2 2 +Created 8 child grid cells + CPU time = 0.00108553 secs + create/ghost percent = 97.5187 2.48128 +balance_grid rcb cell +Balance grid migrated 0 cells + CPU time = 0.000116641 secs + reassign/sort/migrate/ghost percent = 85.2522 0.53326 9.90732 4.30723 + +global nrho 1e10 fnum 1e6 + +species air.species O CO CO2 O2 C +mixture air O O2 vstream 0 1000 -1000 + +mixture air O frac 1.0 +mixture air CO frac 0.0 +mixture air CO2 frac 0.0 +mixture air C frac 0.0 +mixture air O2 frac 0.0 + + +surf_collide 1 cll 300.0 0.5 0.5 0.5 0.5 + +bound_modify zlo collide 1 + +##################################### SURF REACT ADSORB ###################################### +##################################### FACE/BOUNDARY OPTION ################################### + +#surf_react adsorb_test_gs1 adsorb gs sample-GS_1.surf nsync 1 face 1000 6.022e18 O CO +#bound_modify zlo react adsorb_test_gs1 + + +surf_react adsorb_test_gs2 adsorb gs sample-GS_2.surf nsync 1 face 1000 6.022e18 O CO +bound_modify zlo react adsorb_test_gs2 + +########################## BEAM ############################################################ +# Beam at multiple points so that different processors handle the surface collisions + +region circle1 cylinder z 0 -10 1 INF INF + +fix in1 emit/face/file air zhi data.beam beam_area_1 nevery 100 region circle1 twopass + +################################################################################################ + +#dump 2 image all 10 image.*.ppm type type pdiam 0.2 surf proc 0.01 size 512 512 zoom 1.75 gline no 0.005 +#dump_modify 2 pad 4 + +timestep 0.0001 + +stats 10 +stats_style step cpu np nattempt ncoll nscoll nscheck +run 1000 +Memory usage per proc in Mbytes: + particles (ave,min,max) = 0 0 0 + grid (ave,min,max) = 1.51379 1.51379 1.51379 + surf (ave,min,max) = 0 0 0 + modify (ave,min,max) = 0 0 0 + total (ave,min,max) = 1.51379 1.51379 1.51379 +Step CPU Np Natt Ncoll Nscoll Nscheck + 0 0 0 0 0 0 0 + 10 9.906e-06 0 0 0 0 0 + 20 3.8526e-05 0 0 0 0 0 + 30 6.7503e-05 0 0 0 0 0 + 40 7.657e-05 0 0 0 0 0 + 50 8.6626e-05 0 0 0 0 0 + 60 0.000119889 0 0 0 0 0 + 70 0.000132249 0 0 0 0 0 + 80 0.000168524 0 0 0 0 0 + 90 0.000201818 0 0 0 0 0 + 100 0.002038751 3149 0 0 0 0 + 110 0.002333005 3149 0 0 0 0 + 120 0.002623935 3149 0 0 0 0 + 130 0.002903586 3149 0 0 0 0 + 140 0.003169759 3149 0 0 0 0 + 150 0.00350742 3149 0 0 0 0 + 160 0.003790605 3149 0 0 0 0 + 170 0.004052949 3149 0 0 0 0 + 180 0.004317764 3149 0 0 0 0 + 190 0.004609908 3149 0 0 0 0 + 200 0.006249508 3230 0 0 0 0 + 210 0.006536667 3204 0 0 0 0 + 220 0.006823635 3204 0 0 0 0 + 230 0.007178355 3204 0 0 0 0 + 240 0.007496845 3204 0 0 0 0 + 250 0.007806934 3204 0 0 0 0 + 260 0.008079409 3204 0 0 0 0 + 270 0.00840513 3204 0 0 0 0 + 280 0.008715336 3204 0 0 0 0 + 290 0.008986759 3204 0 0 0 0 + 300 0.010594032 3301 0 0 0 0 + 310 0.010883363 3274 0 0 0 0 + 320 0.011181155 3274 0 0 0 0 + 330 0.011708801 3273 0 0 0 0 + 340 0.012036167 3272 0 0 0 0 + 350 0.012378939 3270 0 0 0 0 + 360 0.012682255 3268 0 0 0 0 + 370 0.012968119 3266 0 0 0 0 + 380 0.013242897 3266 0 0 0 0 + 390 0.013530864 3261 0 0 0 0 + 400 0.015158344 3379 0 0 0 0 + 410 0.015538132 3345 0 0 0 0 + 420 0.015847083 3342 0 0 0 0 + 430 0.016139419 3341 0 0 0 0 + 440 0.016428981 3336 0 0 0 0 + 450 0.016769512 3327 0 0 0 0 + 460 0.017056166 3321 0 0 0 0 + 470 0.017341897 3315 0 0 0 0 + 480 0.017640525 3309 0 0 0 0 + 490 0.017917437 3305 0 0 0 0 + 500 0.019554039 3385 0 0 0 0 + 510 0.019879931 3344 0 0 0 0 + 520 0.020162399 3335 0 0 0 0 + 530 0.020447837 3327 0 0 0 0 + 540 0.020768419 3319 0 0 0 0 + 550 0.021156716 3313 0 0 0 0 + 560 0.021447301 3302 0 0 0 0 + 570 0.021749149 3290 0 0 0 0 + 580 0.022036556 3283 0 0 0 0 + 590 0.022418605 3276 0 0 0 0 + 600 0.024177568 3356 0 0 0 0 + 610 0.024489141 3317 0 0 0 0 + 620 0.024829422 3313 0 0 0 0 + 630 0.025138416 3311 0 0 0 0 + 640 0.025430094 3303 0 0 0 0 + 650 0.025770695 3296 0 0 0 0 + 660 0.026066276 3290 0 0 0 0 + 670 0.026348038 3283 0 0 0 0 + 680 0.026645485 3277 0 0 0 0 + 690 0.026920619 3270 0 0 0 0 + 700 0.028634495 3375 0 0 0 0 + 710 0.028942977 3337 0 0 0 0 + 720 0.029225319 3335 0 0 0 0 + 730 0.029521893 3326 0 0 0 0 + 740 0.029830816 3319 0 0 0 0 + 750 0.03014121 3314 0 0 0 0 + 760 0.030425419 3310 0 0 0 0 + 770 0.030723146 3304 0 0 0 0 + 780 0.030999713 3295 0 0 0 0 + 790 0.031276533 3286 0 0 0 0 + 800 0.033056974 3410 0 0 0 0 + 810 0.033367827 3372 0 0 0 0 + 820 0.033675261 3359 0 0 0 0 + 830 0.033955735 3353 0 0 0 0 + 840 0.034235655 3330 0 0 0 0 + 850 0.034548338 3326 0 0 0 0 + 860 0.034892404 3323 0 0 0 0 + 870 0.035181232 3312 0 0 0 0 + 880 0.035512887 3301 0 0 0 0 + 890 0.035813044 3296 0 0 0 0 + 900 0.037409455 3388 0 0 0 0 + 910 0.037739991 3354 0 0 0 0 + 920 0.038030187 3344 0 0 0 0 + 930 0.038315552 3337 0 0 0 0 + 940 0.038620252 3329 0 0 0 0 + 950 0.038931641 3320 0 0 0 0 + 960 0.039278835 3314 0 0 0 0 + 970 0.039699567 3305 0 0 0 0 + 980 0.03999796 3296 0 0 0 0 + 990 0.040378393 3288 0 0 0 0 + 1000 0.042009308 3387 0 0 0 0 +Loop time of 0.0420447 on 1 procs for 1000 steps with 3387 particles +Performance: 23784.187 timesteps/s, 80.557 Mparticle-step/s + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Move | 0.031121 | 0.031121 | 0.031121 | 0.0 | 74.02 +Coll | 0 | 0 | 0 | 0.0 | 0.00 +Sort | 0 | 0 | 0 | 0.0 | 0.00 +Comm | 0.00040146 | 0.00040146 | 0.00040146 | 0.0 | 0.95 +Modify | 0.007106 | 0.007106 | 0.007106 | 0.0 | 16.90 +Output | 0.0028999 | 0.0028999 | 0.0028999 | 0.0 | 6.90 +MPI Sync| 0.00016559 | 0.00016559 | 0.00016559 | 0.0 | 0.39 +Other | | 0.0003508 | | | 0.83 + +Particle moves = 2983471 (2.98M) +Cells touched = 3026752 (3.03M) +Particle comms = 0 (0K) +Boundary collides = 646 (0.646K) +Boundary exits = 434 (0.434K) +SurfColl checks = 0 (0K) +SurfColl occurs = 0 (0K) +Surf reactions = 28166 (28.2K) +Collide attempts = 0 (0K) +Collide occurs = 0 (0K) +Reactions = 0 (0K) +Particles stuck = 0 +Axisymm bad moves = 0 + +Particle-moves/CPUsec/proc: 7.09594e+07 +Particle-moves/step: 2983.47 +Cell-touches/particle/step: 1.01451 +Particle comm iterations/step: 1 +Particle fraction communicated: 0 +Particle fraction colliding with boundary: 0.000216526 +Particle fraction exiting boundary: 0.000145468 +Surface-checks/particle/step: 0 +Surface-collisions/particle/step: 0 +Surf-reactions/particle/step: 0.00944068 +Collision-attempts/particle/step: 0 +Collisions/particle/step: 0 +Reactions/particle/step: 0 + +Surface reaction tallies: + id adsorb_test_gs2 style adsorb #-of-reactions 9 + reaction all: 28166 + reaction O(g) --> O(s): 20991 + reaction O(g) + O(s) --> CO2(g): 2 + reaction O(g) --> CO(s): 6529 + reaction O(g) --> CO(g): 639 + reaction O(g) + O(s) --> O(g) + O(g): 5 + +Particles: 3387 ave 3387 max 3387 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Cells: 8 ave 8 max 8 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +GhostCell: 0 ave 0 max 0 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +EmptyCell: 0 ave 0 max 0 min +Histogram: 1 0 0 0 0 0 0 0 0 0 diff --git a/examples/surf_react_adsorb/log.22Aug26.mpi_1.beam.face.gs_ps b/examples/surf_react_adsorb/log.22Aug26.mpi_1.beam.face.gs_ps new file mode 100644 index 000000000..22ec2ae90 --- /dev/null +++ b/examples/surf_react_adsorb/log.22Aug26.mpi_1.beam.face.gs_ps @@ -0,0 +1,243 @@ +SPARTA (24 Sep 2025) +Running on 1 MPI task(s) +################################################################################ +# beam of particles striking the surface at an inclined angle +# free molecular flow (no collisions) +# +# Note: +# - The "comm/sort” option to the “global” command is used to match MPI runs. +# The "comm/sort" option should not be used for production runs. +################################################################################ + +seed 123456 +dimension 3 +global gridcut 0.0 comm/sort yes + +boundary oo oo so + + +create_box -11 11 -11 11 0 10 +Created orthogonal box = (-11 -11 0) to (11 11 10) +create_grid 2 2 2 +Created 8 child grid cells + CPU time = 0.00113582 secs + create/ghost percent = 97.3038 2.69621 +balance_grid rcb cell +Balance grid migrated 0 cells + CPU time = 0.000135391 secs + reassign/sort/migrate/ghost percent = 84.3919 0.471966 9.88249 5.25367 + +global nrho 1e10 fnum 1e6 + +species air.species O CO CO2 O2 C +mixture air O O2 vstream 0 1000 -1000 + +mixture air O frac 1.0 +mixture air CO frac 0.0 +mixture air CO2 frac 0.0 +mixture air C frac 0.0 +mixture air O2 frac 0.0 + + +surf_collide 1 cll 300.0 0.5 0.5 0.5 0.5 + +bound_modify zlo collide 1 + +##################################### SURF REACT ADSORB ###################################### +##################################### FACE/BOUNDARY OPTION ################################### + +#surf_react adsorb_test_gs_ps1 adsorb gs/ps sample-GS_1.surf sample-PS_1.surf nsync 1 face 1000 6.022e18 O CO +#bound_modify zlo react adsorb_test_gs_ps1 + + +surf_react adsorb_test_gs_ps2 adsorb gs/ps sample-GS_2.surf sample-PS_2.surf nsync 1 face 1000 6.022e18 O CO +bound_modify zlo react adsorb_test_gs_ps2 + +########################## BEAM ############################################################ +# Beam at multiple points so that different processors handle the surface collisions + +region circle1 cylinder z 0 -10 1 INF INF + +fix in1 emit/face/file air zhi data.beam beam_area_1 nevery 100 region circle1 twopass + +################################################################################################ + +#dump 2 image all 10 image.*.ppm type type pdiam 0.2 surf proc 0.01 size 512 512 zoom 1.75 gline no 0.005 +#dump_modify 2 pad 4 + +timestep 0.0001 + +stats 10 +stats_style step cpu np nattempt ncoll nscoll nscheck +run 1000 +Memory usage per proc in Mbytes: + particles (ave,min,max) = 0 0 0 + grid (ave,min,max) = 1.51379 1.51379 1.51379 + surf (ave,min,max) = 0 0 0 + modify (ave,min,max) = 0 0 0 + total (ave,min,max) = 1.51379 1.51379 1.51379 +Step CPU Np Natt Ncoll Nscoll Nscheck + 0 0 0 0 0 0 0 + 10 1.4782e-05 0 0 0 0 0 + 20 4.2685e-05 0 0 0 0 0 + 30 5.368e-05 0 0 0 0 0 + 40 8.5016e-05 0 0 0 0 0 + 50 9.5788e-05 0 0 0 0 0 + 60 0.000105107 0 0 0 0 0 + 70 0.000132017 0 0 0 0 0 + 80 0.00015953 0 0 0 0 0 + 90 0.000185098 0 0 0 0 0 + 100 0.002175931 3131 0 0 0 0 + 110 0.0025427 3132 0 0 0 0 + 120 0.002863449 3132 0 0 0 0 + 130 0.003144381 3132 0 0 0 0 + 140 0.003460943 3132 0 0 0 0 + 150 0.003838593 3132 0 0 0 0 + 160 0.004117498 3132 0 0 0 0 + 170 0.004409659 3132 0 0 0 0 + 180 0.004774107 3133 0 0 0 0 + 190 0.005093368 3135 0 0 0 0 + 200 0.007359159 3964 0 0 0 0 + 210 0.008741466 5808 0 0 0 0 + 220 0.009353671 6036 0 0 0 0 + 230 0.010034448 6032 0 0 0 0 + 240 0.01070906 5975 0 0 0 0 + 250 0.011245484 5896 0 0 0 0 + 260 0.011894299 5831 0 0 0 0 + 270 0.01244337 5767 0 0 0 0 + 280 0.013009928 5682 0 0 0 0 + 290 0.013643728 5594 0 0 0 0 + 300 0.015997565 6060 0 0 0 0 + 310 0.01748596 7746 0 0 0 0 + 320 0.018321725 7924 0 0 0 0 + 330 0.019166901 7818 0 0 0 0 + 340 0.019971727 7647 0 0 0 0 + 350 0.020790574 7447 0 0 0 0 + 360 0.021506491 7245 0 0 0 0 + 370 0.022204023 7057 0 0 0 0 + 380 0.022981215 6876 0 0 0 0 + 390 0.02371119 6701 0 0 0 0 + 400 0.02602577 7019 0 0 0 0 + 410 0.027285073 8107 0 0 0 0 + 420 0.02847323 8645 0 0 0 0 + 430 0.029308571 8506 0 0 0 0 + 440 0.03019066 8280 0 0 0 0 + 450 0.03112234 8023 0 0 0 0 + 460 0.031983238 7778 0 0 0 0 + 470 0.032846051 7551 0 0 0 0 + 480 0.033623892 7299 0 0 0 0 + 490 0.034242281 7071 0 0 0 0 + 500 0.037035997 7630 0 0 0 0 + 510 0.038333214 8514 0 0 0 0 + 520 0.039479681 8858 0 0 0 0 + 530 0.040367972 8800 0 0 0 0 + 540 0.041231125 8600 0 0 0 0 + 550 0.042157418 8347 0 0 0 0 + 560 0.043026125 8114 0 0 0 0 + 570 0.043845939 7852 0 0 0 0 + 580 0.044620248 7619 0 0 0 0 + 590 0.045284789 7396 0 0 0 0 + 600 0.048066478 8013 0 0 0 0 + 610 0.049273669 8723 0 0 0 0 + 620 0.050527281 9132 0 0 0 0 + 630 0.051590045 9065 0 0 0 0 + 640 0.052735645 8849 0 0 0 0 + 650 0.053713089 8574 0 0 0 0 + 660 0.054508393 8289 0 0 0 0 + 670 0.055248453 8027 0 0 0 0 + 680 0.056057241 7778 0 0 0 0 + 690 0.056841626 7531 0 0 0 0 + 700 0.05961315 8365 0 0 0 0 + 710 0.061122776 9546 0 0 0 0 + 720 0.062190471 9483 0 0 0 0 + 730 0.06315935 9231 0 0 0 0 + 740 0.06408605 8976 0 0 0 0 + 750 0.06504031 8703 0 0 0 0 + 760 0.065927624 8420 0 0 0 0 + 770 0.066772627 8120 0 0 0 0 + 780 0.067520334 7849 0 0 0 0 + 790 0.068233945 7608 0 0 0 0 + 800 0.070906806 8283 0 0 0 0 + 810 0.072196817 9089 0 0 0 0 + 820 0.073343196 9384 0 0 0 0 + 830 0.074506884 9290 0 0 0 0 + 840 0.075349582 9042 0 0 0 0 + 850 0.076328983 8784 0 0 0 0 + 860 0.077210811 8523 0 0 0 0 + 870 0.078084244 8219 0 0 0 0 + 880 0.078921655 7944 0 0 0 0 + 890 0.079728425 7703 0 0 0 0 + 900 0.082435939 8395 0 0 0 0 + 910 0.084128652 9691 0 0 0 0 + 920 0.085284117 9652 0 0 0 0 + 930 0.086283603 9414 0 0 0 0 + 940 0.087251035 9120 0 0 0 0 + 950 0.088208438 8843 0 0 0 0 + 960 0.089100824 8556 0 0 0 0 + 970 0.089999414 8268 0 0 0 0 + 980 0.09084549 7989 0 0 0 0 + 990 0.091635759 7719 0 0 0 0 + 1000 0.094160461 8175 0 0 0 0 +Loop time of 0.0942055 on 1 procs for 1000 steps with 8175 particles +Performance: 10615.087 timesteps/s, 86.778 Mparticle-step/s + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Move | 0.069604 | 0.069604 | 0.069604 | 0.0 | 73.89 +Coll | 0 | 0 | 0 | 0.0 | 0.00 +Sort | 0 | 0 | 0 | 0.0 | 0.00 +Comm | 0.00085276 | 0.00085276 | 0.00085276 | 0.0 | 0.91 +Modify | 0.0085642 | 0.0085642 | 0.0085642 | 0.0 | 9.09 +Output | 0.0070079 | 0.0070079 | 0.0070079 | 0.0 | 7.44 +MPI Sync| 0.00024524 | 0.00024524 | 0.00024524 | 0.0 | 0.26 +Other | | 0.007932 | | | 8.42 + +Particle moves = 6686588 (6.69M) +Cells touched = 6752526 (6.75M) +Particle comms = 0 (0K) +Boundary collides = 625 (0.625K) +Boundary exits = 18944 (18.9K) +SurfColl checks = 0 (0K) +SurfColl occurs = 0 (0K) +Surf reactions = 28211 (28.2K) +Collide attempts = 0 (0K) +Collide occurs = 0 (0K) +Reactions = 0 (0K) +Particles stuck = 0 +Axisymm bad moves = 0 + +Particle-moves/CPUsec/proc: 7.09787e+07 +Particle-moves/step: 6686.59 +Cell-touches/particle/step: 1.00986 +Particle comm iterations/step: 1 +Particle fraction communicated: 0 +Particle fraction colliding with boundary: 9.34707e-05 +Particle fraction exiting boundary: 0.00283313 +Surface-checks/particle/step: 0 +Surface-collisions/particle/step: 0 +Surf-reactions/particle/step: 0.00421904 +Collision-attempts/particle/step: 0 +Collisions/particle/step: 0 +Reactions/particle/step: 0 + +Surface reaction tallies: + id adsorb_test_gs_ps2 style adsorb #-of-reactions 14 + reaction all: 52838 + reaction O(g) --> O(s): 20981 + reaction O(g) --> CO(s): 6605 + reaction O(g) --> CO(g): 625 + reaction O(s) --> O(g): 13946 + reaction CO(s) --> CO(g): 7191 + reaction 2O(s) + C(b) --> CO2(g): 2148 + reaction O(s) + C(b) --> CO(s): 1327 + reaction C(b) --> C(g): 15 + +Particles: 8175 ave 8175 max 8175 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Cells: 8 ave 8 max 8 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +GhostCell: 0 ave 0 max 0 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +EmptyCell: 0 ave 0 max 0 min +Histogram: 1 0 0 0 0 0 0 0 0 0 diff --git a/examples/surf_react_adsorb/log.22Aug26.mpi_1.beam.face.ps b/examples/surf_react_adsorb/log.22Aug26.mpi_1.beam.face.ps new file mode 100644 index 000000000..138b2b57a Binary files /dev/null and b/examples/surf_react_adsorb/log.22Aug26.mpi_1.beam.face.ps differ diff --git a/examples/surf_react_adsorb/log.22Aug26.mpi_1.beam.surf.gs b/examples/surf_react_adsorb/log.22Aug26.mpi_1.beam.surf.gs new file mode 100644 index 000000000..c873fac0d --- /dev/null +++ b/examples/surf_react_adsorb/log.22Aug26.mpi_1.beam.surf.gs @@ -0,0 +1,259 @@ +SPARTA (24 Sep 2025) +Running on 1 MPI task(s) +################################################################################ +# beam of particles striking the surface at an inclined angle +# free molecular flow (no collisions) +# +# Note: +# - The "comm/sort” option to the “global” command is used to match MPI runs. +# The "comm/sort" option should not be used for production runs. +################################################################################ + +seed 123456 +dimension 3 +global gridcut 0.0 comm/sort yes + +boundary oo oo oo + + +create_box -11 11 -11 11 0 10 +Created orthogonal box = (-11 -11 0) to (11 11 10) +create_grid 2 2 2 +Created 8 child grid cells + CPU time = 0.00116704 secs + create/ghost percent = 97.3621 2.63788 +balance_grid rcb cell +Balance grid migrated 0 cells + CPU time = 9.2887e-05 secs + reassign/sort/migrate/ghost percent = 82.1568 0.573815 12.0071 5.26231 + +global nrho 1e10 fnum 1e6 + +species air.species O CO CO2 O2 C +mixture air O O2 vstream 0 1000 -1000 + +mixture air O frac 1.0 +mixture air CO frac 0.0 +mixture air CO2 frac 0.0 +mixture air C frac 0.0 +mixture air O2 frac 0.0 + + +surf_collide 1 cll 300.0 0.5 0.5 0.5 0.5 + +read_surf base_plate.surf + 12 triangles + -11 11 xlo xhi + -11 11 ylo yhi + 0 1 zlo zhi + 1 min triangle edge length + 11 min triangle area + 4 0 = cells overlapping surfs, overlap cells with unmarked corner pts + 4 0 4 = cells outside/inside/overlapping surfs + 4 = surf cells with 1,2,etc splits + 4356 4356 = cell-wise and global flow volume + CPU time = 0.000625412 secs + read/check/sort/surf2grid/ghost/inout/particle percent = 8.40614 28.1674 0.302041 56.8513 6.27314 7.26097 0.022705 + surf2grid time = 0.000355555 secs + map/comm1/comm2/comm3/comm4/split percent = 29.2371 6.48845 2.71463 2.8575 20.2843 37.5377 + +##################################### SURF REACT ADSORB ###################################### +##################################### SURF OPTION ############################################ + +#surf_react adsorb_test_gs1 adsorb gs sample-GS_1.surf nsync 1 surf 1000 6.022e18 O CO +#surf_modify all collide 1 react adsorb_test_gs1 + +surf_react adsorb_test_gs2 adsorb gs sample-GS_2.surf nsync 1 surf 1000 6.022e18 O CO +surf_modify all collide 1 react adsorb_test_gs2 + +########################## BEAM ############################################################ +# Beam at multiple points so that different processors handle the surface collisions + +region circle2 cylinder z 6 -10 1 INF INF +region circle3 cylinder z -6 -10 1 INF INF + +fix in2 emit/face/file air zhi data.beam beam_area_2 nevery 100 region circle2 twopass +fix in3 emit/face/file air zhi data.beam beam_area_3 nevery 100 region circle3 twopass + +################################################################################################ + +#dump 2 image all 10 image.*.ppm type type pdiam 0.2 surf proc 0.01 size 512 512 zoom 1.75 gline no 0.005 +#dump_modify 2 pad 4 + +timestep 0.0001 + +stats 10 +stats_style step cpu np nattempt ncoll nscoll nscheck +run 1000 +Memory usage per proc in Mbytes: + particles (ave,min,max) = 0 0 0 + grid (ave,min,max) = 1.51379 1.51379 1.51379 + surf (ave,min,max) = 0.00151062 0.00151062 0.00151062 + modify (ave,min,max) = 0 0 0 + total (ave,min,max) = 1.5153 1.5153 1.5153 +Step CPU Np Natt Ncoll Nscoll Nscheck + 0 0 0 0 0 0 0 + 10 1.7374e-05 0 0 0 0 0 + 20 4.8229e-05 0 0 0 0 0 + 30 7.8491e-05 0 0 0 0 0 + 40 0.000107757 0 0 0 0 0 + 50 0.000139655 0 0 0 0 0 + 60 0.000167327 0 0 0 0 0 + 70 0.00019857 0 0 0 0 0 + 80 0.000256784 0 0 0 0 0 + 90 0.000290148 0 0 0 0 0 + 100 0.00268705 6216 0 0 0 0 + 110 0.003382528 6216 0 0 0 0 + 120 0.004088701 6216 0 0 0 0 + 130 0.004918386 6216 0 0 0 0 + 140 0.005739727 6216 0 0 0 0 + 150 0.006893626 6216 0 0 0 49488 + 160 0.01084866 6216 0 0 0 49728 + 170 0.014819581 6216 0 0 0 49728 + 180 0.018812655 6216 0 0 0 49728 + 190 0.024388528 188 0 0 6133 50568 + 200 0.025728792 6432 0 0 0 1080 + 210 0.026627987 6432 0 0 0 1072 + 220 0.02749882 6432 0 0 0 1080 + 230 0.028265944 6432 0 0 0 1064 + 240 0.029052449 6432 0 0 0 1040 + 250 0.030226546 6432 0 0 0 51184 + 260 0.034200726 6432 0 0 0 51312 + 270 0.038236526 6427 0 0 0 51200 + 280 0.042222287 6423 0 0 0 51088 + 290 0.049496015 324 0 0 6212 51904 + 300 0.051482382 6606 0 0 0 1744 + 310 0.052668206 6604 0 0 0 1656 + 320 0.053815506 6599 0 0 0 1608 + 330 0.055019291 6595 0 0 0 1584 + 340 0.056172131 6587 0 0 0 1568 + 350 0.057879135 6584 0 0 0 51816 + 360 0.063494161 6575 0 0 0 51928 + 370 0.069046138 6569 0 0 0 51800 + 380 0.07467771 6560 0 0 0 51648 + 390 0.082242796 420 0 0 6239 52464 + 400 0.084199384 6633 0 0 0 2096 + 410 0.08541291 6618 0 0 0 1992 + 420 0.086632961 6604 0 0 0 1928 + 430 0.087786242 6589 0 0 0 1840 + 440 0.088933717 6576 0 0 0 1776 + 450 0.090660985 6560 0 0 0 51488 + 460 0.096172041 6546 0 0 0 51544 + 470 0.10197149 6535 0 0 0 51400 + 480 0.1072203 6525 0 0 0 51224 + 490 0.11327255 445 0 0 6154 51728 + 500 0.11482124 6716 0 0 0 1920 + 510 0.11582284 6700 0 0 0 1808 + 520 0.11674046 6692 0 0 0 1720 + 530 0.11763488 6682 0 0 0 1688 + 540 0.11855342 6671 0 0 0 1576 + 550 0.11991309 6657 0 0 0 51856 + 560 0.12428431 6643 0 0 0 52000 + 570 0.12863988 6628 0 0 0 51864 + 580 0.13285456 6608 0 0 0 51712 + 590 0.13854661 472 0 0 6218 52312 + 600 0.14007937 6769 0 0 0 1936 + 610 0.14105059 6753 0 0 0 1856 + 620 0.14192519 6745 0 0 0 1776 + 630 0.14282802 6727 0 0 0 1680 + 640 0.14368836 6712 0 0 0 1576 + 650 0.14498568 6697 0 0 0 52304 + 660 0.14904443 6691 0 0 0 52376 + 670 0.15324774 6676 0 0 0 52208 + 680 0.15739822 6659 0 0 0 52024 + 690 0.16325702 470 0 0 6275 52688 + 700 0.16477754 6701 0 0 0 1928 + 710 0.16572805 6681 0 0 0 1832 + 720 0.1666681 6668 0 0 0 1736 + 730 0.16806141 6654 0 0 0 1648 + 740 0.16898307 6638 0 0 0 1568 + 750 0.17031971 6623 0 0 0 51624 + 760 0.1744397 6605 0 0 0 51752 + 770 0.17854852 6593 0 0 0 51592 + 780 0.18258663 6579 0 0 0 51480 + 790 0.18816435 449 0 0 6204 52096 + 800 0.18967692 6626 0 0 0 1920 + 810 0.19063454 6610 0 0 0 1856 + 820 0.19152028 6599 0 0 0 1728 + 830 0.19234537 6580 0 0 0 1648 + 840 0.19359969 6564 0 0 0 1520 + 850 0.1949392 6546 0 0 0 51048 + 860 0.19907621 6535 0 0 0 51160 + 870 0.2030909 6512 0 0 0 51008 + 880 0.20710713 6498 0 0 0 50896 + 890 0.21307935 449 0 0 6141 51624 + 900 0.21459233 6628 0 0 0 1920 + 910 0.21546683 6615 0 0 0 1800 + 920 0.21629751 6603 0 0 0 1728 + 930 0.21714136 6590 0 0 0 1656 + 940 0.21798448 6577 0 0 0 1584 + 950 0.21928043 6557 0 0 0 51304 + 960 0.2232973 6542 0 0 0 51360 + 970 0.22727855 6529 0 0 0 51216 + 980 0.23150473 6519 0 0 0 51144 + 990 0.23750588 471 0 0 6165 52048 + 1000 0.23982323 6669 0 0 0 2312 +Loop time of 0.239878 on 1 procs for 1000 steps with 6669 particles +Performance: 4168.791 timesteps/s, 27.802 Mparticle-step/s + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Move | 0.2199 | 0.2199 | 0.2199 | 0.0 | 91.67 +Coll | 0 | 0 | 0 | 0.0 | 0.00 +Sort | 0 | 0 | 0 | 0.0 | 0.00 +Comm | 0.00066918 | 0.00066918 | 0.00066918 | 0.0 | 0.28 +Modify | 0.014353 | 0.014353 | 0.014353 | 0.0 | 5.98 +Output | 0.0033003 | 0.0033003 | 0.0033003 | 0.0 | 1.38 +MPI Sync| 0.00028946 | 0.00028946 | 0.00028946 | 0.0 | 0.12 +Other | | 0.001366 | | | 0.57 + +Particle moves = 5397929 (5.4M) +Cells touched = 5456156 (5.46M) +Particle comms = 0 (0K) +Boundary collides = 0 (0K) +Boundary exits = 934 (0.934K) +SurfColl checks = 19766792 (19.8M) +SurfColl occurs = 56634 (56.6K) +Surf reactions = 56634 (56.6K) +Collide attempts = 0 (0K) +Collide occurs = 0 (0K) +Reactions = 0 (0K) +Particles stuck = 0 +Axisymm bad moves = 0 + +Particle-moves/CPUsec/proc: 2.25028e+07 +Particle-moves/step: 5397.93 +Cell-touches/particle/step: 1.01079 +Particle comm iterations/step: 1 +Particle fraction communicated: 0 +Particle fraction colliding with boundary: 0 +Particle fraction exiting boundary: 0.000173029 +Surface-checks/particle/step: 3.66192 +Surface-collisions/particle/step: 0.0104918 +Surf-reactions/particle/step: 0.0104918 +Collision-attempts/particle/step: 0 +Collisions/particle/step: 0 +Reactions/particle/step: 0 + +Surface reaction tallies: + id adsorb_test_gs2 style adsorb #-of-reactions 9 + reaction all: 56634 + reaction O(g) --> O(s): 42304 + reaction O(g) + O(s) --> CO2(g): 8 + reaction O(g) --> CO(s): 13008 + reaction O(g) --> CO(g): 1291 + reaction O(g) + O(s) --> O(g) + O(g): 23 + +Particles: 6669 ave 6669 max 6669 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Cells: 8 ave 8 max 8 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +GhostCell: 0 ave 0 max 0 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +EmptyCell: 0 ave 0 max 0 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Surfs: 12 ave 12 max 12 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +GhostSurf: 0 ave 0 max 0 min +Histogram: 1 0 0 0 0 0 0 0 0 0 diff --git a/examples/surf_react_adsorb/log.22Aug26.mpi_1.beam.surf.gs_ps b/examples/surf_react_adsorb/log.22Aug26.mpi_1.beam.surf.gs_ps new file mode 100644 index 000000000..6b73d6216 --- /dev/null +++ b/examples/surf_react_adsorb/log.22Aug26.mpi_1.beam.surf.gs_ps @@ -0,0 +1,263 @@ +SPARTA (24 Sep 2025) +Running on 1 MPI task(s) +################################################################################ +# beam of particles striking the surface at an inclined angle +# free molecular flow (no collisions) +# +# Note: +# - The "comm/sort” option to the “global” command is used to match MPI runs. +# The "comm/sort" option should not be used for production runs. +################################################################################ + +seed 123456 +dimension 3 +global gridcut 0.0 comm/sort yes + +boundary oo oo oo + + +create_box -11 11 -11 11 0 10 +Created orthogonal box = (-11 -11 0) to (11 11 10) +create_grid 2 2 2 +Created 8 child grid cells + CPU time = 0.00110884 secs + create/ghost percent = 96.354 3.64596 +balance_grid rcb cell +Balance grid migrated 0 cells + CPU time = 9.6451e-05 secs + reassign/sort/migrate/ghost percent = 82.3092 0.497662 11.6204 5.57278 + +global nrho 1e10 fnum 1e6 + +species air.species O CO CO2 O2 C +mixture air O O2 vstream 0 1000 -1000 + +mixture air O frac 1.0 +mixture air CO frac 0.0 +mixture air CO2 frac 0.0 +mixture air C frac 0.0 +mixture air O2 frac 0.0 + + +surf_collide 1 cll 300.0 0.5 0.5 0.5 0.5 + +read_surf base_plate.surf + 12 triangles + -11 11 xlo xhi + -11 11 ylo yhi + 0 1 zlo zhi + 1 min triangle edge length + 11 min triangle area + 4 0 = cells overlapping surfs, overlap cells with unmarked corner pts + 4 0 4 = cells outside/inside/overlapping surfs + 4 = surf cells with 1,2,etc splits + 4356 4356 = cell-wise and global flow volume + CPU time = 0.000582107 secs + read/check/sort/surf2grid/ghost/inout/particle percent = 8.75767 28.0586 0.280533 55.6999 7.20331 5.12998 0.0135714 + surf2grid time = 0.000324233 secs + map/comm1/comm2/comm3/comm4/split percent = 30.5453 7.60595 3.19369 3.20695 5.51609 45.876 + +##################################### SURF REACT ADSORB ###################################### +##################################### SURF OPTION ############################################ + +#surf_react adsorb_test_gs_ps1 adsorb gs/ps sample-GS_1.surf sample-PS_1.surf nsync 1 surf 1000 6.022e18 O CO +#surf_modify all collide 1 react adsorb_test_gs_ps1 + +surf_react adsorb_test_gs_ps2 adsorb gs/ps sample-GS_2.surf sample-PS_2.surf nsync 1 surf 1000 6.022e18 O CO +surf_modify all collide 1 react adsorb_test_gs_ps2 + +########################## BEAM ############################################################ +# Beam at multiple points so that different processors handle the surface collisions + +region circle2 cylinder z 6 -10 1 INF INF +region circle3 cylinder z -6 -10 1 INF INF + +fix in2 emit/face/file air zhi data.beam beam_area_2 nevery 100 region circle2 twopass +fix in3 emit/face/file air zhi data.beam beam_area_3 nevery 100 region circle3 twopass + +################################################################################################ + +#dump 2 image all 10 image.*.ppm type type pdiam 0.2 surf proc 0.01 size 512 512 zoom 1.75 gline no 0.005 +#dump_modify 2 pad 4 + +timestep 0.0001 + +stats 10 +stats_style step cpu np nattempt ncoll nscoll nscheck +run 1000 +Memory usage per proc in Mbytes: + particles (ave,min,max) = 0 0 0 + grid (ave,min,max) = 1.51379 1.51379 1.51379 + surf (ave,min,max) = 0.00151062 0.00151062 0.00151062 + modify (ave,min,max) = 0 0 0 + total (ave,min,max) = 1.5153 1.5153 1.5153 +Step CPU Np Natt Ncoll Nscoll Nscheck + 0 0 0 0 0 0 0 + 10 4.7291e-05 0 0 0 0 0 + 20 9.6381e-05 0 0 0 0 0 + 30 0.000145084 0 0 0 0 0 + 40 0.000191625 0 0 0 0 0 + 50 0.000236295 0 0 0 0 0 + 60 0.000282244 0 0 0 0 0 + 70 0.000367591 0 0 0 0 0 + 80 0.000417634 0 0 0 0 0 + 90 0.000462207 0 0 0 0 0 + 100 0.00292883 6294 0 0 0 0 + 110 0.003687422 6296 0 0 0 16 + 120 0.004498456 6296 0 0 0 24 + 130 0.005218679 6297 0 0 0 16 + 140 0.00601644 6296 0 0 0 16 + 150 0.007141501 6297 0 0 0 50184 + 160 0.011157171 6298 0 0 0 50376 + 170 0.015271037 6298 0 0 0 50376 + 180 0.019389102 6299 0 0 0 50376 + 190 0.025942152 1575 0 0 6215 51216 + 200 0.031398599 11574 0 0 0 41584 + 210 0.036095216 12004 0 0 0 46096 + 220 0.040596355 11963 0 0 0 45832 + 230 0.044988596 11857 0 0 0 44344 + 240 0.049431064 11739 0 0 0 41368 + 250 0.054156416 11611 0 0 0 87008 + 260 0.061161797 11463 0 0 0 82112 + 270 0.067898356 11295 0 0 0 77152 + 280 0.074335122 11076 0 0 0 72808 + 290 0.083901058 7079 0 0 6166 70288 + 300 0.090806306 15151 0 0 0 51384 + 310 0.096760095 15164 0 0 0 52000 + 320 0.10219377 14894 0 0 0 50176 + 330 0.1073681 14525 0 0 0 47488 + 340 0.11233624 14146 0 0 0 44048 + 350 0.11763211 13807 0 0 0 90792 + 360 0.12562873 13398 0 0 0 86624 + 370 0.13324594 13009 0 0 0 82296 + 380 0.14051738 12631 0 0 0 78248 + 390 0.14961752 7135 0 0 6250 75512 + 400 0.15735002 16276 0 0 0 55160 + 410 0.16652015 16493 0 0 0 57928 + 420 0.17352511 16178 0 0 0 56208 + 430 0.18283438 15797 0 0 1 53504 + 440 0.1926035 15353 0 0 0 49832 + 450 0.20314171 14923 0 0 0 95544 + 460 0.2148742 14518 0 0 0 90840 + 470 0.22341659 14078 0 0 0 85920 + 480 0.23130997 13633 0 0 0 81296 + 490 0.24077491 8580 0 0 6186 77984 + 500 0.24722896 15922 0 0 0 46992 + 510 0.25376024 17021 0 0 0 57016 + 520 0.26010394 16827 0 0 0 56448 + 530 0.26673796 16374 0 0 0 53824 + 540 0.27251501 15887 0 0 1 50216 + 550 0.27857347 15380 0 0 0 95368 + 560 0.28700123 14903 0 0 0 90984 + 570 0.29548227 14450 0 0 0 86368 + 580 0.30326233 13993 0 0 0 81736 + 590 0.31375106 9387 0 0 6129 78304 + 600 0.3222605 17206 0 0 0 56256 + 610 0.32915863 17499 0 0 0 58984 + 620 0.335532 17216 0 0 0 57848 + 630 0.34170608 16810 0 0 0 55384 + 640 0.34748179 16284 0 0 0 51344 + 650 0.35335508 15810 0 0 0 97064 + 660 0.36174355 15362 0 0 0 92280 + 670 0.36984005 14861 0 0 0 87240 + 680 0.37777648 14370 0 0 0 82752 + 690 0.38763454 9116 0 0 6211 79464 + 700 0.39459824 16983 0 0 0 51088 + 710 0.401324 17697 0 0 0 59360 + 720 0.40774092 17419 0 0 0 58448 + 730 0.41403362 16980 0 0 0 55808 + 740 0.42001802 16465 0 0 0 52040 + 750 0.42676009 15932 0 0 0 97304 + 760 0.43527357 15462 0 0 0 92784 + 770 0.44348266 14992 0 0 0 87928 + 780 0.45142725 14503 0 0 0 83024 + 790 0.46147879 9310 0 0 6142 79400 + 800 0.46949713 17960 0 0 0 59320 + 810 0.47619199 18028 0 0 0 60704 + 820 0.48334355 17615 0 0 0 58440 + 830 0.48962509 17107 0 0 0 55456 + 840 0.49552768 16540 0 0 0 51248 + 850 0.50147321 16030 0 0 0 96480 + 860 0.50996065 15504 0 0 0 91480 + 870 0.51819297 15019 0 0 0 86544 + 880 0.52737179 14521 0 0 0 82144 + 890 0.537712 10200 0 0 6167 78960 + 900 0.5454618 18098 0 0 0 60320 + 910 0.55232881 17973 0 0 0 60824 + 920 0.55879356 17526 0 0 0 58040 + 930 0.56494586 16991 0 0 0 54552 + 940 0.57077913 16476 0 0 0 50640 + 950 0.57677736 15959 0 0 0 95984 + 960 0.58545492 15460 0 0 0 91328 + 970 0.59360211 15002 0 0 0 86232 + 980 0.60134779 14525 0 0 0 82008 + 990 0.61089806 9496 0 0 6156 78896 + 1000 0.61777765 16986 0 0 0 50528 +Loop time of 0.61782 on 1 procs for 1000 steps with 16986 particles +Performance: 1618.595 timesteps/s, 27.493 Mparticle-step/s + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Move | 0.56756 | 0.56756 | 0.56756 | 0.0 | 91.86 +Coll | 0 | 0 | 0 | 0.0 | 0.00 +Sort | 0 | 0 | 0 | 0.0 | 0.00 +Comm | 0.0021993 | 0.0021993 | 0.0021993 | 0.0 | 0.36 +Modify | 0.013981 | 0.013981 | 0.013981 | 0.0 | 2.26 +Output | 0.012762 | 0.012762 | 0.012762 | 0.0 | 2.07 +MPI Sync| 0.00079615 | 0.00079615 | 0.00079615 | 0.0 | 0.13 +Other | | 0.02053 | | | 3.32 + +Particle moves = 12330796 (12.3M) +Cells touched = 12433974 (12.4M) +Particle comms = 0 (0K) +Boundary collides = 0 (0K) +Boundary exits = 36301 (36.3K) +SurfColl checks = 54867688 (54.9M) +SurfColl occurs = 56528 (56.5K) +Surf reactions = 56527 (56.5K) +Collide attempts = 0 (0K) +Collide occurs = 0 (0K) +Reactions = 0 (0K) +Particles stuck = 0 +Axisymm bad moves = 0 + +Particle-moves/CPUsec/proc: 1.99586e+07 +Particle-moves/step: 12330.8 +Cell-touches/particle/step: 1.00837 +Particle comm iterations/step: 1 +Particle fraction communicated: 0 +Particle fraction colliding with boundary: 0 +Particle fraction exiting boundary: 0.00294393 +Surface-checks/particle/step: 4.44965 +Surface-collisions/particle/step: 0.00458429 +Surf-reactions/particle/step: 0.00458421 +Collision-attempts/particle/step: 0 +Collisions/particle/step: 0 +Reactions/particle/step: 0 + +Surface reaction tallies: + id adsorb_test_gs_ps2 style adsorb #-of-reactions 14 + reaction all: 104475 + reaction O(g) --> O(s): 42186 + reaction O(g) --> CO(s): 13052 + reaction O(g) --> CO(g): 1285 + reaction C(g) --> C(b): 4 + reaction O(s) --> O(g): 24230 + reaction CO(s) --> CO(g): 13584 + reaction 2O(s) + C(b) --> CO2(g): 7780 + reaction O(s) + C(b) --> CO(s): 2234 + reaction C(b) --> C(g): 120 + +Particles: 16986 ave 16986 max 16986 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Cells: 8 ave 8 max 8 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +GhostCell: 0 ave 0 max 0 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +EmptyCell: 0 ave 0 max 0 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Surfs: 12 ave 12 max 12 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +GhostSurf: 0 ave 0 max 0 min +Histogram: 1 0 0 0 0 0 0 0 0 0 diff --git a/examples/surf_react_adsorb/log.22Aug26.mpi_1.beam.surf.ps b/examples/surf_react_adsorb/log.22Aug26.mpi_1.beam.surf.ps new file mode 100644 index 000000000..303152028 Binary files /dev/null and b/examples/surf_react_adsorb/log.22Aug26.mpi_1.beam.surf.ps differ diff --git a/examples/surf_react_adsorb/log.22Aug26.mpi_4.beam.face.gs b/examples/surf_react_adsorb/log.22Aug26.mpi_4.beam.face.gs new file mode 100644 index 000000000..0907598a2 --- /dev/null +++ b/examples/surf_react_adsorb/log.22Aug26.mpi_4.beam.face.gs @@ -0,0 +1,241 @@ +SPARTA (24 Sep 2025) +Running on 4 MPI task(s) +################################################################################ +# beam of particles striking the surface at an inclined angle +# free molecular flow (no collisions) +# +# Note: +# - The "comm/sort” option to the “global” command is used to match MPI runs. +# The "comm/sort" option should not be used for production runs. +################################################################################ + +seed 123456 +dimension 3 +global gridcut 0.0 comm/sort yes + +boundary oo oo so + + +create_box -11 11 -11 11 0 10 +Created orthogonal box = (-11 -11 0) to (11 11 10) +create_grid 2 2 2 +WARNING: Could not acquire nearby ghost cells b/c grid partition is not clumped (/home/user/sparta/src/grid.cpp:486) +Created 8 child grid cells + CPU time = 0.00145738 secs + create/ghost percent = 90.0735 9.92648 +balance_grid rcb cell +Balance grid migrated 4 cells + CPU time = 0.000362139 secs + reassign/sort/migrate/ghost percent = 66.2276 2.19142 10.3466 21.2344 + +global nrho 1e10 fnum 1e6 + +species air.species O CO CO2 O2 C +mixture air O O2 vstream 0 1000 -1000 + +mixture air O frac 1.0 +mixture air CO frac 0.0 +mixture air CO2 frac 0.0 +mixture air C frac 0.0 +mixture air O2 frac 0.0 + + +surf_collide 1 cll 300.0 0.5 0.5 0.5 0.5 + +bound_modify zlo collide 1 + +##################################### SURF REACT ADSORB ###################################### +##################################### FACE/BOUNDARY OPTION ################################### + +#surf_react adsorb_test_gs1 adsorb gs sample-GS_1.surf nsync 1 face 1000 6.022e18 O CO +#bound_modify zlo react adsorb_test_gs1 + + +surf_react adsorb_test_gs2 adsorb gs sample-GS_2.surf nsync 1 face 1000 6.022e18 O CO +bound_modify zlo react adsorb_test_gs2 + +########################## BEAM ############################################################ +# Beam at multiple points so that different processors handle the surface collisions + +region circle1 cylinder z 0 -10 1 INF INF + +fix in1 emit/face/file air zhi data.beam beam_area_1 nevery 100 region circle1 twopass + +################################################################################################ + +#dump 2 image all 10 image.*.ppm type type pdiam 0.2 surf proc 0.01 size 512 512 zoom 1.75 gline no 0.005 +#dump_modify 2 pad 4 + +timestep 0.0001 + +stats 10 +stats_style step cpu np nattempt ncoll nscoll nscheck +run 1000 +Memory usage per proc in Mbytes: + particles (ave,min,max) = 0 0 0 + grid (ave,min,max) = 1.51379 1.51379 1.51379 + surf (ave,min,max) = 0 0 0 + modify (ave,min,max) = 0 0 0 + total (ave,min,max) = 1.51379 1.51379 1.51379 +Step CPU Np Natt Ncoll Nscoll Nscheck + 0 0 0 0 0 0 0 + 10 0.000176694 0 0 0 0 0 + 20 0.000327012 0 0 0 0 0 + 30 0.000498606 0 0 0 0 0 + 40 0.0006214 0 0 0 0 0 + 50 0.000778711 0 0 0 0 0 + 60 0.000896651 0 0 0 0 0 + 70 0.001015864 0 0 0 0 0 + 80 0.001130213 0 0 0 0 0 + 90 0.00127682 0 0 0 0 0 + 100 0.003177437 3141 0 0 0 0 + 110 0.003584792 3141 0 0 0 0 + 120 0.004002611 3141 0 0 0 0 + 130 0.004297821 3141 0 0 0 0 + 140 0.004595496 3141 0 0 0 0 + 150 0.004862795 3141 0 0 0 0 + 160 0.005099541 3141 0 0 0 0 + 170 0.005382073 3141 0 0 0 0 + 180 0.005710775 3141 0 0 0 0 + 190 0.007271319 3141 0 0 0 0 + 200 0.008598137 3213 0 0 0 0 + 210 0.009115735 3187 0 0 0 0 + 220 0.00957046 3187 0 0 0 0 + 230 0.009949352 3187 0 0 0 0 + 240 0.010311606 3187 0 0 0 0 + 250 0.010698143 3187 0 0 0 0 + 260 0.011037813 3187 0 0 0 0 + 270 0.011314185 3187 0 0 0 0 + 280 0.011651977 3187 0 0 0 0 + 290 0.01194862 3187 0 0 0 0 + 300 0.013111442 3304 0 0 0 0 + 310 0.013497513 3274 0 0 0 0 + 320 0.013819228 3272 0 0 0 0 + 330 0.01411953 3272 0 0 0 0 + 340 0.014496838 3271 0 0 0 0 + 350 0.014842899 3270 0 0 0 0 + 360 0.015149289 3268 0 0 0 0 + 370 0.01540583 3267 0 0 0 0 + 380 0.015766973 3265 0 0 0 0 + 390 0.016042216 3263 0 0 0 0 + 400 0.017178609 3354 0 0 0 0 + 410 0.017594432 3325 0 0 0 0 + 420 0.017899836 3322 0 0 0 0 + 430 0.018217843 3318 0 0 0 0 + 440 0.018719146 3314 0 0 0 0 + 450 0.019245263 3311 0 0 0 0 + 460 0.019777146 3302 0 0 0 0 + 470 0.020173204 3294 0 0 0 0 + 480 0.020422914 3290 0 0 0 0 + 490 0.02076718 3283 0 0 0 0 + 500 0.021808763 3340 0 0 0 0 + 510 0.02212597 3306 0 0 0 0 + 520 0.022573524 3299 0 0 0 0 + 530 0.022913572 3292 0 0 0 0 + 540 0.023256225 3288 0 0 0 0 + 550 0.023640712 3282 0 0 0 0 + 560 0.02400112 3276 0 0 0 0 + 570 0.024359219 3270 0 0 0 0 + 580 0.024739879 3270 0 0 0 0 + 590 0.025046327 3261 0 0 0 0 + 600 0.026078083 3403 0 0 0 0 + 610 0.026525886 3366 0 0 0 0 + 620 0.026857782 3361 0 0 0 0 + 630 0.027173369 3356 0 0 0 0 + 640 0.027504164 3348 0 0 0 0 + 650 0.027817657 3342 0 0 0 0 + 660 0.028094057 3333 0 0 0 0 + 670 0.028360598 3324 0 0 0 0 + 680 0.02873432 3319 0 0 0 0 + 690 0.029012862 3311 0 0 0 0 + 700 0.030041332 3403 0 0 0 0 + 710 0.030505113 3369 0 0 0 0 + 720 0.030855768 3365 0 0 0 0 + 730 0.031172213 3357 0 0 0 0 + 740 0.0314825 3351 0 0 0 0 + 750 0.031842863 3346 0 0 0 0 + 760 0.032101801 3340 0 0 0 0 + 770 0.032364309 3328 0 0 0 0 + 780 0.032692538 3318 0 0 0 0 + 790 0.032993145 3313 0 0 0 0 + 800 0.034327281 3446 0 0 0 0 + 810 0.034754402 3393 0 0 0 0 + 820 0.035089584 3387 0 0 0 0 + 830 0.035422395 3382 0 0 0 0 + 840 0.035802768 3376 0 0 0 0 + 850 0.036081358 3369 0 0 0 0 + 860 0.036361994 3360 0 0 0 0 + 870 0.036680547 3354 0 0 0 0 + 880 0.037051293 3347 0 0 0 0 + 890 0.03738311 3341 0 0 0 0 + 900 0.038529525 3359 0 0 0 0 + 910 0.03890522 3316 0 0 0 0 + 920 0.039186576 3309 0 0 0 0 + 930 0.039509704 3299 0 0 0 0 + 940 0.03982448 3291 0 0 0 0 + 950 0.040138196 3281 0 0 0 0 + 960 0.040409795 3272 0 0 0 0 + 970 0.040746389 3263 0 0 0 0 + 980 0.041015896 3257 0 0 0 0 + 990 0.041275833 3251 0 0 0 0 + 1000 0.042277372 3393 0 0 0 0 +Loop time of 0.0422895 on 4 procs for 1000 steps with 3393 particles +Performance: 23646.559 timesteps/s, 80.233 Mparticle-step/s + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Move | 0.0026514 | 0.009688 | 0.016873 | 7.1 | 22.91 +Coll | 0 | 0 | 0 | 0.0 | 0.00 +Sort | 0 | 0 | 0 | 0.0 | 0.00 +Comm | 0.010018 | 0.010323 | 0.010666 | 0.2 | 24.41 +Modify | 7.3928e-05 | 0.0025649 | 0.0050964 | 4.9 | 6.07 +Output | 0.0011582 | 0.0014406 | 0.0022417 | 1.2 | 3.41 +MPI Sync| 0.0041678 | 0.01458 | 0.024291 | 8.0 | 34.48 +Other | | 0.003693 | | | 8.73 + +Particle moves = 2956450 (2.96M) +Cells touched = 3026859 (3.03M) +Particle comms = 14716 (14.7K) +Boundary collides = 629 (0.629K) +Boundary exits = 402 (0.402K) +SurfColl checks = 0 (0K) +SurfColl occurs = 0 (0K) +Surf reactions = 28229 (28.2K) +Collide attempts = 0 (0K) +Collide occurs = 0 (0K) +Reactions = 0 (0K) +Particles stuck = 0 +Axisymm bad moves = 0 + +Particle-moves/CPUsec/proc: 1.74775e+07 +Particle-moves/step: 2956.45 +Cell-touches/particle/step: 1.02382 +Particle comm iterations/step: 1.431 +Particle fraction communicated: 0.00497759 +Particle fraction colliding with boundary: 0.000212755 +Particle fraction exiting boundary: 0.000135974 +Surface-checks/particle/step: 0 +Surface-collisions/particle/step: 0 +Surf-reactions/particle/step: 0.00954828 +Collision-attempts/particle/step: 0 +Collisions/particle/step: 0 +Reactions/particle/step: 0 + +Surface reaction tallies: + id adsorb_test_gs2 style adsorb #-of-reactions 9 + reaction all: 28229 + reaction O(g) --> O(s): 20941 + reaction O(g) + O(s) --> CO2(g): 4 + reaction O(g) --> CO(s): 6659 + reaction O(g) --> CO(g): 617 + reaction O(g) + O(s) --> O(g) + O(g): 8 + +Particles: 848.25 ave 1643 max 59 min +Histogram: 2 0 0 0 0 0 0 0 0 2 +Cells: 2 ave 2 max 2 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +GhostCell: 6 ave 6 max 6 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +EmptyCell: 6 ave 6 max 6 min +Histogram: 4 0 0 0 0 0 0 0 0 0 diff --git a/examples/surf_react_adsorb/log.22Aug26.mpi_4.beam.face.gs_ps b/examples/surf_react_adsorb/log.22Aug26.mpi_4.beam.face.gs_ps new file mode 100644 index 000000000..0255ad26c --- /dev/null +++ b/examples/surf_react_adsorb/log.22Aug26.mpi_4.beam.face.gs_ps @@ -0,0 +1,244 @@ +SPARTA (24 Sep 2025) +Running on 4 MPI task(s) +################################################################################ +# beam of particles striking the surface at an inclined angle +# free molecular flow (no collisions) +# +# Note: +# - The "comm/sort” option to the “global” command is used to match MPI runs. +# The "comm/sort" option should not be used for production runs. +################################################################################ + +seed 123456 +dimension 3 +global gridcut 0.0 comm/sort yes + +boundary oo oo so + + +create_box -11 11 -11 11 0 10 +Created orthogonal box = (-11 -11 0) to (11 11 10) +create_grid 2 2 2 +WARNING: Could not acquire nearby ghost cells b/c grid partition is not clumped (/home/user/sparta/src/grid.cpp:486) +Created 8 child grid cells + CPU time = 0.00131395 secs + create/ghost percent = 89.9515 10.0485 +balance_grid rcb cell +Balance grid migrated 4 cells + CPU time = 0.000379487 secs + reassign/sort/migrate/ghost percent = 68.5552 0.446656 12.5946 18.4035 + +global nrho 1e10 fnum 1e6 + +species air.species O CO CO2 O2 C +mixture air O O2 vstream 0 1000 -1000 + +mixture air O frac 1.0 +mixture air CO frac 0.0 +mixture air CO2 frac 0.0 +mixture air C frac 0.0 +mixture air O2 frac 0.0 + + +surf_collide 1 cll 300.0 0.5 0.5 0.5 0.5 + +bound_modify zlo collide 1 + +##################################### SURF REACT ADSORB ###################################### +##################################### FACE/BOUNDARY OPTION ################################### + +#surf_react adsorb_test_gs_ps1 adsorb gs/ps sample-GS_1.surf sample-PS_1.surf nsync 1 face 1000 6.022e18 O CO +#bound_modify zlo react adsorb_test_gs_ps1 + + +surf_react adsorb_test_gs_ps2 adsorb gs/ps sample-GS_2.surf sample-PS_2.surf nsync 1 face 1000 6.022e18 O CO +bound_modify zlo react adsorb_test_gs_ps2 + +########################## BEAM ############################################################ +# Beam at multiple points so that different processors handle the surface collisions + +region circle1 cylinder z 0 -10 1 INF INF + +fix in1 emit/face/file air zhi data.beam beam_area_1 nevery 100 region circle1 twopass + +################################################################################################ + +#dump 2 image all 10 image.*.ppm type type pdiam 0.2 surf proc 0.01 size 512 512 zoom 1.75 gline no 0.005 +#dump_modify 2 pad 4 + +timestep 0.0001 + +stats 10 +stats_style step cpu np nattempt ncoll nscoll nscheck +run 1000 +Memory usage per proc in Mbytes: + particles (ave,min,max) = 0 0 0 + grid (ave,min,max) = 1.51379 1.51379 1.51379 + surf (ave,min,max) = 0 0 0 + modify (ave,min,max) = 0 0 0 + total (ave,min,max) = 1.51379 1.51379 1.51379 +Step CPU Np Natt Ncoll Nscoll Nscheck + 0 0 0 0 0 0 0 + 10 0.000194154 0 0 0 0 0 + 20 0.000406375 0 0 0 0 0 + 30 0.000601468 0 0 0 0 0 + 40 0.000959766 0 0 0 0 0 + 50 0.001213284 0 0 0 0 0 + 60 0.001420841 0 0 0 0 0 + 70 0.00162493 0 0 0 0 0 + 80 0.001829453 0 0 0 0 0 + 90 0.002057886 0 0 0 0 0 + 100 0.003802558 3116 0 0 0 0 + 110 0.005461563 3117 0 0 0 0 + 120 0.005964299 3117 0 0 0 0 + 130 0.006422576 3117 0 0 0 0 + 140 0.006896394 3117 0 0 0 0 + 150 0.007414791 3117 0 0 0 0 + 160 0.007846158 3117 0 0 0 0 + 170 0.00823024 3117 0 0 0 0 + 180 0.00980393 3118 0 0 0 0 + 190 0.010210485 3120 0 0 0 0 + 200 0.011954127 3838 0 0 0 0 + 210 0.013604035 5782 0 0 0 0 + 220 0.014537712 6017 0 0 0 0 + 230 0.015113028 5991 0 0 0 0 + 240 0.015588751 5940 0 0 0 0 + 250 0.016109682 5880 0 0 0 0 + 260 0.016566429 5807 0 0 0 0 + 270 0.017097362 5732 0 0 0 0 + 280 0.01755982 5663 0 0 0 0 + 290 0.018070397 5587 0 0 0 0 + 300 0.020229887 6467 0 0 0 0 + 310 0.022384683 7839 0 0 0 0 + 320 0.023308789 7885 0 0 0 0 + 330 0.024089848 7752 0 0 0 0 + 340 0.024747042 7593 0 0 0 0 + 350 0.025589717 7383 0 0 0 0 + 360 0.026395792 7205 0 0 0 0 + 370 0.02719427 7018 0 0 0 0 + 380 0.02802796 6822 0 0 0 0 + 390 0.028551611 6630 0 0 0 0 + 400 0.030225969 7220 0 0 0 0 + 410 0.031630368 8134 0 0 0 0 + 420 0.032666434 8530 0 0 0 0 + 430 0.033389684 8469 0 0 0 0 + 440 0.034049118 8282 0 0 0 0 + 450 0.034607138 8055 0 0 0 0 + 460 0.035155334 7819 0 0 0 0 + 470 0.035648307 7590 0 0 0 0 + 480 0.036176761 7367 0 0 0 0 + 490 0.036658911 7139 0 0 0 0 + 500 0.038374218 7863 0 0 0 0 + 510 0.039786756 9200 0 0 0 0 + 520 0.040590264 9197 0 0 0 0 + 530 0.041317266 8962 0 0 0 0 + 540 0.041959393 8706 0 0 0 0 + 550 0.042670279 8441 0 0 0 0 + 560 0.043277211 8201 0 0 0 0 + 570 0.043804909 7921 0 0 0 0 + 580 0.044387688 7683 0 0 0 0 + 590 0.045025803 7439 0 0 0 0 + 600 0.04680941 8164 0 0 0 0 + 610 0.048270583 9366 0 0 0 0 + 620 0.049116437 9403 0 0 0 0 + 630 0.049762163 9185 0 0 0 0 + 640 0.050377062 8908 0 0 0 0 + 650 0.051015327 8653 0 0 0 0 + 660 0.051578847 8331 0 0 0 0 + 670 0.052108396 8075 0 0 0 0 + 680 0.052676001 7829 0 0 0 0 + 690 0.053237522 7564 0 0 0 0 + 700 0.055220827 8338 0 0 0 0 + 710 0.056633137 9571 0 0 0 0 + 720 0.057418007 9541 0 0 0 0 + 730 0.059074683 9297 0 0 0 0 + 740 0.059722902 9049 0 0 0 0 + 750 0.060325 8745 0 0 0 0 + 760 0.060986124 8435 0 0 0 0 + 770 0.061606549 8162 0 0 0 0 + 780 0.062225905 7900 0 0 0 0 + 790 0.062737442 7643 0 0 0 0 + 800 0.064412803 8377 0 0 0 0 + 810 0.065798722 9551 0 0 0 0 + 820 0.066933214 9479 0 0 0 0 + 830 0.067657225 9243 0 0 0 0 + 840 0.068249295 8932 0 0 0 0 + 850 0.068980368 8636 0 0 0 0 + 860 0.069667982 8337 0 0 0 0 + 870 0.070246564 8043 0 0 0 0 + 880 0.070756879 7779 0 0 0 0 + 890 0.071360377 7530 0 0 0 0 + 900 0.072991015 8059 0 0 0 0 + 910 0.074629884 9535 0 0 0 0 + 920 0.0753731 9495 0 0 0 0 + 930 0.076026856 9245 0 0 0 0 + 940 0.076604242 8970 0 0 0 0 + 950 0.077239284 8683 0 0 0 0 + 960 0.077756088 8400 0 0 0 0 + 970 0.07830435 8108 0 0 0 0 + 980 0.078797728 7849 0 0 0 0 + 990 0.079352512 7577 0 0 0 0 + 1000 0.081025202 8161 0 0 0 0 +Loop time of 0.0810641 on 4 procs for 1000 steps with 8161 particles +Performance: 12335.912 timesteps/s, 100.673 Mparticle-step/s + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Move | 0.012009 | 0.019383 | 0.026858 | 5.3 | 23.91 +Coll | 0 | 0 | 0 | 0.0 | 0.00 +Sort | 0 | 0 | 0 | 0.0 | 0.00 +Comm | 0.011086 | 0.011531 | 0.011904 | 0.3 | 14.22 +Modify | 9.2595e-05 | 0.002439 | 0.0048476 | 4.7 | 3.01 +Output | 0.0081418 | 0.0091123 | 0.011981 | 1.7 | 11.24 +MPI Sync| 0.0059612 | 0.01821 | 0.02917 | 7.9 | 22.46 +Other | | 0.02039 | | | 25.15 + +Particle moves = 6656819 (6.66M) +Cells touched = 6769407 (6.77M) +Particle comms = 24016 (24K) +Boundary collides = 625 (0.625K) +Boundary exits = 19254 (19.3K) +SurfColl checks = 0 (0K) +SurfColl occurs = 0 (0K) +Surf reactions = 28340 (28.3K) +Collide attempts = 0 (0K) +Collide occurs = 0 (0K) +Reactions = 0 (0K) +Particles stuck = 0 +Axisymm bad moves = 0 + +Particle-moves/CPUsec/proc: 2.05295e+07 +Particle-moves/step: 6656.82 +Cell-touches/particle/step: 1.01691 +Particle comm iterations/step: 1.828 +Particle fraction communicated: 0.00360773 +Particle fraction colliding with boundary: 9.38887e-05 +Particle fraction exiting boundary: 0.00289237 +Surface-checks/particle/step: 0 +Surface-collisions/particle/step: 0 +Surf-reactions/particle/step: 0.00425729 +Collision-attempts/particle/step: 0 +Collisions/particle/step: 0 +Reactions/particle/step: 0 + +Surface reaction tallies: + id adsorb_test_gs_ps2 style adsorb #-of-reactions 14 + reaction all: 53106 + reaction O(g) --> O(s): 21237 + reaction O(g) --> CO(s): 6478 + reaction O(g) --> CO(g): 625 + reaction O(s) --> O(g): 14873 + reaction CO(s) --> CO(g): 6734 + reaction 2O(s) + C(b) --> CO2(g): 2028 + reaction O(s) + C(b) --> CO(s): 1119 + reaction C(b) --> C(g): 12 + +Particles: 2040.25 ave 2846 max 1210 min +Histogram: 2 0 0 0 0 0 0 0 0 2 +Cells: 2 ave 2 max 2 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +GhostCell: 6 ave 6 max 6 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +EmptyCell: 6 ave 6 max 6 min +Histogram: 4 0 0 0 0 0 0 0 0 0 diff --git a/examples/surf_react_adsorb/log.22Aug26.mpi_4.beam.face.ps b/examples/surf_react_adsorb/log.22Aug26.mpi_4.beam.face.ps new file mode 100644 index 000000000..41b436fec Binary files /dev/null and b/examples/surf_react_adsorb/log.22Aug26.mpi_4.beam.face.ps differ diff --git a/examples/surf_react_adsorb/log.22Aug26.mpi_4.beam.surf.gs b/examples/surf_react_adsorb/log.22Aug26.mpi_4.beam.surf.gs new file mode 100644 index 000000000..56df14c31 --- /dev/null +++ b/examples/surf_react_adsorb/log.22Aug26.mpi_4.beam.surf.gs @@ -0,0 +1,260 @@ +SPARTA (24 Sep 2025) +Running on 4 MPI task(s) +################################################################################ +# beam of particles striking the surface at an inclined angle +# free molecular flow (no collisions) +# +# Note: +# - The "comm/sort” option to the “global” command is used to match MPI runs. +# The "comm/sort" option should not be used for production runs. +################################################################################ + +seed 123456 +dimension 3 +global gridcut 0.0 comm/sort yes + +boundary oo oo oo + + +create_box -11 11 -11 11 0 10 +Created orthogonal box = (-11 -11 0) to (11 11 10) +create_grid 2 2 2 +WARNING: Could not acquire nearby ghost cells b/c grid partition is not clumped (/home/user/sparta/src/grid.cpp:486) +Created 8 child grid cells + CPU time = 0.0017971 secs + create/ghost percent = 91.0973 8.90274 +balance_grid rcb cell +Balance grid migrated 4 cells + CPU time = 0.000372161 secs + reassign/sort/migrate/ghost percent = 67.7825 0.613982 11.2508 20.3528 + +global nrho 1e10 fnum 1e6 + +species air.species O CO CO2 O2 C +mixture air O O2 vstream 0 1000 -1000 + +mixture air O frac 1.0 +mixture air CO frac 0.0 +mixture air CO2 frac 0.0 +mixture air C frac 0.0 +mixture air O2 frac 0.0 + + +surf_collide 1 cll 300.0 0.5 0.5 0.5 0.5 + +read_surf base_plate.surf + 12 triangles + -11 11 xlo xhi + -11 11 ylo yhi + 0 1 zlo zhi + 1 min triangle edge length + 11 min triangle area + 4 0 = cells overlapping surfs, overlap cells with unmarked corner pts + 4 0 4 = cells outside/inside/overlapping surfs + 4 = surf cells with 1,2,etc splits + 4356 4356 = cell-wise and global flow volume + CPU time = 0.000799601 secs + read/check/sort/surf2grid/ghost/inout/particle percent = 5.75475 33.7525 0.343296 51.606 8.54351 7.19696 0.125438 + surf2grid time = 0.000412642 secs + map/comm1/comm2/comm3/comm4/split percent = 36.8421 13.0658 5.47133 7.02691 18.3539 14.8288 + +##################################### SURF REACT ADSORB ###################################### +##################################### SURF OPTION ############################################ + +#surf_react adsorb_test_gs1 adsorb gs sample-GS_1.surf nsync 1 surf 1000 6.022e18 O CO +#surf_modify all collide 1 react adsorb_test_gs1 + +surf_react adsorb_test_gs2 adsorb gs sample-GS_2.surf nsync 1 surf 1000 6.022e18 O CO +surf_modify all collide 1 react adsorb_test_gs2 + +########################## BEAM ############################################################ +# Beam at multiple points so that different processors handle the surface collisions + +region circle2 cylinder z 6 -10 1 INF INF +region circle3 cylinder z -6 -10 1 INF INF + +fix in2 emit/face/file air zhi data.beam beam_area_2 nevery 100 region circle2 twopass +fix in3 emit/face/file air zhi data.beam beam_area_3 nevery 100 region circle3 twopass + +################################################################################################ + +#dump 2 image all 10 image.*.ppm type type pdiam 0.2 surf proc 0.01 size 512 512 zoom 1.75 gline no 0.005 +#dump_modify 2 pad 4 + +timestep 0.0001 + +stats 10 +stats_style step cpu np nattempt ncoll nscoll nscheck +run 1000 +Memory usage per proc in Mbytes: + particles (ave,min,max) = 0 0 0 + grid (ave,min,max) = 1.51379 1.51379 1.51379 + surf (ave,min,max) = 0.00151062 0.00151062 0.00151062 + modify (ave,min,max) = 0 0 0 + total (ave,min,max) = 1.5153 1.5153 1.5153 +Step CPU Np Natt Ncoll Nscoll Nscheck + 0 0 0 0 0 0 0 + 10 0.000137864 0 0 0 0 0 + 20 0.000274756 0 0 0 0 0 + 30 0.000412627 0 0 0 0 0 + 40 0.000541888 0 0 0 0 0 + 50 0.000829103 0 0 0 0 0 + 60 0.001045218 0 0 0 0 0 + 70 0.001246048 0 0 0 0 0 + 80 0.001497987 0 0 0 0 0 + 90 0.001760178 0 0 0 0 0 + 100 0.004343575 6270 0 0 0 0 + 110 0.005414411 6270 0 0 0 0 + 120 0.00641353 6270 0 0 0 0 + 130 0.007366954 6270 0 0 0 0 + 140 0.008321646 6270 0 0 0 0 + 150 0.009790516 6270 0 0 0 49928 + 160 0.014513643 6270 0 0 0 50160 + 170 0.019233217 6270 0 0 0 50160 + 180 0.024028423 6270 0 0 0 50160 + 190 0.031465869 181 0 0 6176 50864 + 200 0.034209028 6450 0 0 0 1024 + 210 0.035292864 6450 0 0 0 1008 + 220 0.036410767 6450 0 0 0 1024 + 230 0.037523471 6450 0 0 0 1016 + 240 0.038564999 6450 0 0 0 1000 + 250 0.040117406 6450 0 0 0 51336 + 260 0.045069072 6450 0 0 0 51432 + 270 0.049947977 6449 0 0 0 51336 + 280 0.054817433 6448 0 0 0 51248 + 290 0.060995606 312 0 0 6233 51936 + 300 0.062533784 6573 0 0 0 1656 + 310 0.063665993 6568 0 0 0 1584 + 320 0.064876543 6562 0 0 0 1528 + 330 0.065604482 6555 0 0 0 1472 + 340 0.066275919 6550 0 0 0 1416 + 350 0.067256016 6543 0 0 0 51584 + 360 0.069855727 6535 0 0 0 51672 + 370 0.072387326 6527 0 0 0 51520 + 380 0.074906841 6520 0 0 0 51368 + 390 0.078373806 412 0 0 6211 52160 + 400 0.079427881 6614 0 0 0 1984 + 410 0.080208767 6602 0 0 0 1880 + 420 0.081010259 6599 0 0 0 1824 + 430 0.081688563 6587 0 0 0 1784 + 440 0.082320199 6572 0 0 0 1728 + 450 0.083252341 6559 0 0 0 51392 + 460 0.085860588 6555 0 0 0 51496 + 470 0.088400632 6540 0 0 0 51328 + 480 0.090946662 6530 0 0 0 51224 + 490 0.09435115 453 0 0 6141 51792 + 500 0.095384456 6664 0 0 0 2048 + 510 0.096041204 6648 0 0 0 2008 + 520 0.09672998 6636 0 0 0 1936 + 530 0.097408892 6624 0 0 0 1856 + 540 0.098049483 6606 0 0 0 1760 + 550 0.098958611 6586 0 0 0 51616 + 560 0.10152543 6576 0 0 0 51728 + 570 0.10416765 6560 0 0 0 51568 + 580 0.10671101 6546 0 0 0 51400 + 590 0.11012873 462 0 0 6171 52104 + 600 0.11118094 6681 0 0 0 2168 + 610 0.11190403 6670 0 0 0 2048 + 620 0.11252463 6655 0 0 0 1928 + 630 0.11326655 6643 0 0 0 1856 + 640 0.11393542 6629 0 0 0 1720 + 650 0.11481945 6614 0 0 0 51672 + 660 0.11741351 6597 0 0 0 51648 + 670 0.11996215 6585 0 0 0 51544 + 680 0.12257015 6573 0 0 0 51344 + 690 0.12609812 484 0 0 6196 52088 + 700 0.12715104 6715 0 0 0 2088 + 710 0.12791471 6706 0 0 0 1968 + 720 0.12867701 6689 0 0 0 1904 + 730 0.12938005 6675 0 0 0 1864 + 740 0.13022623 6654 0 0 0 1704 + 750 0.13115827 6629 0 0 0 51680 + 760 0.13386975 6616 0 0 0 51744 + 770 0.13637484 6599 0 0 0 51600 + 780 0.13894225 6586 0 0 0 51432 + 790 0.14325682 458 0 0 6205 52016 + 800 0.14433656 6689 0 0 0 1984 + 810 0.14508294 6666 0 0 0 1920 + 820 0.14585499 6648 0 0 0 1784 + 830 0.14651944 6628 0 0 0 1720 + 840 0.14715817 6613 0 0 0 1600 + 850 0.14808249 6603 0 0 0 51504 + 860 0.15069034 6590 0 0 0 51624 + 870 0.15334745 6576 0 0 0 51488 + 880 0.15606628 6565 0 0 0 51368 + 890 0.15959123 481 0 0 6183 52152 + 900 0.16067269 6582 0 0 0 2144 + 910 0.16133542 6561 0 0 0 2088 + 920 0.16218244 6551 0 0 0 2016 + 930 0.16290252 6536 0 0 0 1896 + 940 0.16372776 6513 0 0 0 1768 + 950 0.16460089 6504 0 0 0 50680 + 960 0.16734574 6492 0 0 0 50792 + 970 0.16988921 6475 0 0 0 50608 + 980 0.17232908 6461 0 0 0 50472 + 990 0.17583875 477 0 0 6060 51112 + 1000 0.17691915 6645 0 0 0 2040 +Loop time of 0.176973 on 4 procs for 1000 steps with 6645 particles +Performance: 5650.588 timesteps/s, 37.548 Mparticle-step/s + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Move | 0.0023472 | 0.065535 | 0.13964 | 24.8 | 37.03 +Coll | 0 | 0 | 0 | 0.0 | 0.00 +Sort | 0 | 0 | 0 | 0.0 | 0.00 +Comm | 0.008541 | 0.0089149 | 0.0093901 | 0.4 | 5.04 +Modify | 0.00014889 | 0.004435 | 0.0093131 | 6.5 | 2.51 +Output | 0.0017069 | 0.0027988 | 0.0060288 | 3.5 | 1.58 +MPI Sync| 0.0096508 | 0.087962 | 0.15681 | 23.3 | 49.70 +Other | | 0.007327 | | | 4.14 + +Particle moves = 5351732 (5.35M) +Cells touched = 5447142 (5.45M) +Particle comms = 723 (0.723K) +Boundary collides = 0 (0K) +Boundary exits = 941 (0.941K) +SurfColl checks = 19762256 (19.8M) +SurfColl occurs = 56462 (56.5K) +Surf reactions = 56462 (56.5K) +Collide attempts = 0 (0K) +Collide occurs = 0 (0K) +Reactions = 0 (0K) +Particles stuck = 0 +Axisymm bad moves = 0 + +Particle-moves/CPUsec/proc: 7.56011e+06 +Particle-moves/step: 5351.73 +Cell-touches/particle/step: 1.01783 +Particle comm iterations/step: 1.455 +Particle fraction communicated: 0.000135096 +Particle fraction colliding with boundary: 0 +Particle fraction exiting boundary: 0.000175831 +Surface-checks/particle/step: 3.69268 +Surface-collisions/particle/step: 0.0105502 +Surf-reactions/particle/step: 0.0105502 +Collision-attempts/particle/step: 0 +Collisions/particle/step: 0 +Reactions/particle/step: 0 + +Surface reaction tallies: + id adsorb_test_gs2 style adsorb #-of-reactions 9 + reaction all: 56462 + reaction O(g) --> O(s): 41986 + reaction O(g) + O(s) --> CO2(g): 9 + reaction O(g) --> CO(s): 13164 + reaction O(g) --> CO(g): 1275 + reaction O(g) + O(s) --> O(g) + O(g): 28 + +Particles: 1661.25 ave 3281 max 51 min +Histogram: 2 0 0 0 0 0 0 0 0 2 +Cells: 2 ave 2 max 2 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +GhostCell: 6 ave 6 max 6 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +EmptyCell: 6 ave 6 max 6 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +Surfs: 12 ave 12 max 12 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +GhostSurf: 0 ave 0 max 0 min +Histogram: 4 0 0 0 0 0 0 0 0 0 diff --git a/examples/surf_react_adsorb/log.22Aug26.mpi_4.beam.surf.gs_ps b/examples/surf_react_adsorb/log.22Aug26.mpi_4.beam.surf.gs_ps new file mode 100644 index 000000000..d21b26c84 --- /dev/null +++ b/examples/surf_react_adsorb/log.22Aug26.mpi_4.beam.surf.gs_ps @@ -0,0 +1,264 @@ +SPARTA (24 Sep 2025) +Running on 4 MPI task(s) +################################################################################ +# beam of particles striking the surface at an inclined angle +# free molecular flow (no collisions) +# +# Note: +# - The "comm/sort” option to the “global” command is used to match MPI runs. +# The "comm/sort" option should not be used for production runs. +################################################################################ + +seed 123456 +dimension 3 +global gridcut 0.0 comm/sort yes + +boundary oo oo oo + + +create_box -11 11 -11 11 0 10 +Created orthogonal box = (-11 -11 0) to (11 11 10) +create_grid 2 2 2 +WARNING: Could not acquire nearby ghost cells b/c grid partition is not clumped (/home/user/sparta/src/grid.cpp:486) +Created 8 child grid cells + CPU time = 0.00125255 secs + create/ghost percent = 94.6611 5.33889 +balance_grid rcb cell +Balance grid migrated 4 cells + CPU time = 0.000404383 secs + reassign/sort/migrate/ghost percent = 74.4732 0.54305 10.2737 14.7101 + +global nrho 1e10 fnum 1e6 + +species air.species O CO CO2 O2 C +mixture air O O2 vstream 0 1000 -1000 + +mixture air O frac 1.0 +mixture air CO frac 0.0 +mixture air CO2 frac 0.0 +mixture air C frac 0.0 +mixture air O2 frac 0.0 + + +surf_collide 1 cll 300.0 0.5 0.5 0.5 0.5 + +read_surf base_plate.surf + 12 triangles + -11 11 xlo xhi + -11 11 ylo yhi + 0 1 zlo zhi + 1 min triangle edge length + 11 min triangle area + 4 0 = cells overlapping surfs, overlap cells with unmarked corner pts + 4 0 4 = cells outside/inside/overlapping surfs + 4 = surf cells with 1,2,etc splits + 4356 4356 = cell-wise and global flow volume + CPU time = 0.000689887 secs + read/check/sort/surf2grid/ghost/inout/particle percent = 5.63629 27.352 0.3115 55.8131 10.8871 8.0877 0.135819 + surf2grid time = 0.000385047 secs + map/comm1/comm2/comm3/comm4/split percent = 34.1364 7.22016 4.94096 3.21805 31.491 14.9145 + +##################################### SURF REACT ADSORB ###################################### +##################################### SURF OPTION ############################################ + +#surf_react adsorb_test_gs_ps1 adsorb gs/ps sample-GS_1.surf sample-PS_1.surf nsync 1 surf 1000 6.022e18 O CO +#surf_modify all collide 1 react adsorb_test_gs_ps1 + +surf_react adsorb_test_gs_ps2 adsorb gs/ps sample-GS_2.surf sample-PS_2.surf nsync 1 surf 1000 6.022e18 O CO +surf_modify all collide 1 react adsorb_test_gs_ps2 + +########################## BEAM ############################################################ +# Beam at multiple points so that different processors handle the surface collisions + +region circle2 cylinder z 6 -10 1 INF INF +region circle3 cylinder z -6 -10 1 INF INF + +fix in2 emit/face/file air zhi data.beam beam_area_2 nevery 100 region circle2 twopass +fix in3 emit/face/file air zhi data.beam beam_area_3 nevery 100 region circle3 twopass + +################################################################################################ + +#dump 2 image all 10 image.*.ppm type type pdiam 0.2 surf proc 0.01 size 512 512 zoom 1.75 gline no 0.005 +#dump_modify 2 pad 4 + +timestep 0.0001 + +stats 10 +stats_style step cpu np nattempt ncoll nscoll nscheck +run 1000 +Memory usage per proc in Mbytes: + particles (ave,min,max) = 0 0 0 + grid (ave,min,max) = 1.51379 1.51379 1.51379 + surf (ave,min,max) = 0.00151062 0.00151062 0.00151062 + modify (ave,min,max) = 0 0 0 + total (ave,min,max) = 1.5153 1.5153 1.5153 +Step CPU Np Natt Ncoll Nscoll Nscheck + 0 0 0 0 0 0 0 + 10 0.000224036 0 0 0 0 0 + 20 0.000500325 0 0 0 0 0 + 30 0.000656573 0 0 0 0 0 + 40 0.000814442 0 0 0 0 0 + 50 0.000993114 0 0 0 0 0 + 60 0.001504357 0 0 0 0 0 + 70 0.00188182 0 0 0 0 0 + 80 0.002311785 0 0 0 0 0 + 90 0.002656883 0 0 0 0 0 + 100 0.005471142 6301 0 0 0 0 + 110 0.007605491 6303 0 0 0 16 + 120 0.008421247 6303 0 0 0 16 + 130 0.009053036 6305 0 0 0 24 + 140 0.009778189 6304 0 0 0 24 + 150 0.010695277 6304 0 0 0 50216 + 160 0.013581006 6304 0 0 0 50440 + 170 0.016154246 6305 0 0 0 50432 + 180 0.018846876 6305 0 0 0 50432 + 190 0.023540313 1733 0 0 6209 51232 + 200 0.02686686 11612 0 0 0 41248 + 210 0.029350447 12081 0 0 0 46016 + 220 0.031573904 12059 0 0 0 46048 + 230 0.033689482 11947 0 0 0 44448 + 240 0.035566956 11831 0 0 0 41272 + 250 0.037593442 11692 0 0 0 87368 + 260 0.041521312 11553 0 0 0 82680 + 270 0.045404247 11362 0 0 0 78008 + 280 0.048895877 11160 0 0 0 74144 + 290 0.054167341 6891 0 0 6231 71744 + 300 0.057804113 14691 0 0 0 48912 + 310 0.060548761 15167 0 0 0 52392 + 320 0.063318129 15066 0 0 0 51856 + 330 0.065922387 14764 0 0 0 49512 + 340 0.068341169 14388 0 0 0 46104 + 350 0.070835679 14009 0 0 0 92344 + 360 0.075063488 13626 0 0 0 88256 + 370 0.079778442 13253 0 0 0 84072 + 380 0.083500656 12846 0 0 0 79560 + 390 0.08906784 7822 0 0 6252 76368 + 400 0.093420987 16217 0 0 0 53088 + 410 0.09656533 16589 0 0 0 57392 + 420 0.099515547 16395 0 0 0 56352 + 430 0.10254917 15997 0 0 0 53592 + 440 0.10503694 15537 0 0 0 49568 + 450 0.10772395 15077 0 0 0 95416 + 460 0.11176682 14641 0 0 0 91008 + 470 0.11580786 14212 0 0 0 86288 + 480 0.11983539 13743 0 0 0 81928 + 490 0.12532567 8608 0 0 6243 78608 + 500 0.12895382 16562 0 0 0 53288 + 510 0.13181428 16511 0 0 0 53752 + 520 0.1344315 16147 0 0 0 51368 + 530 0.13703732 16153 0 0 0 51568 + 540 0.13956061 15958 0 0 0 50216 + 550 0.14273403 15581 0 0 0 96392 + 560 0.14722585 15153 0 0 0 92160 + 570 0.15134218 14699 0 0 0 87792 + 580 0.15550908 14253 0 0 0 83544 + 590 0.16073947 9174 0 0 6150 80656 + 600 0.16465203 17840 0 0 0 60264 + 610 0.167616 17849 0 0 0 61528 + 620 0.1703088 17458 0 0 0 59400 + 630 0.17287801 16940 0 0 0 56000 + 640 0.17570458 16408 0 0 0 51728 + 650 0.18285647 15920 0 0 0 97264 + 660 0.18714427 15412 0 0 0 92664 + 670 0.19123538 14924 0 0 0 87672 + 680 0.1951066 14448 0 0 0 82600 + 690 0.20078913 9897 0 0 6218 79112 + 700 0.20473163 17512 0 0 0 55624 + 710 0.20779582 17633 0 0 0 57824 + 720 0.21066693 17295 0 0 0 56336 + 730 0.21381189 16799 0 0 0 53312 + 740 0.21743778 16289 0 0 0 49600 + 750 0.22024478 15798 0 0 0 95264 + 760 0.22499027 15299 0 0 0 90568 + 770 0.22889382 14815 0 0 0 86264 + 780 0.23277836 14330 0 0 0 81792 + 790 0.23776466 8938 0 0 6189 78656 + 800 0.24192129 17320 0 0 0 54536 + 810 0.24483788 17530 0 0 0 57112 + 820 0.24809189 17405 0 0 0 57360 + 830 0.25096426 17002 0 0 0 55208 + 840 0.25350701 16515 0 0 0 51936 + 850 0.25606767 16046 0 0 0 97840 + 860 0.26012526 15557 0 0 0 93336 + 870 0.2639586 15028 0 0 0 88528 + 880 0.26779215 14556 0 0 0 83752 + 890 0.27391928 10213 0 0 6197 80272 + 900 0.2786567 17534 0 0 0 56552 + 910 0.28160127 17521 0 0 0 56992 + 920 0.28613302 17435 0 0 0 57232 + 930 0.28894242 17046 0 0 0 55016 + 940 0.29183853 16599 0 0 0 51232 + 950 0.29876505 16129 0 0 0 96920 + 960 0.3104408 15610 0 0 0 92832 + 970 0.31459111 15155 0 0 0 88232 + 980 0.31924264 14649 0 0 0 83784 + 990 0.32476776 9967 0 0 6154 80600 + 1000 0.32900958 18362 0 0 0 62080 +Loop time of 0.329065 on 4 procs for 1000 steps with 18362 particles +Performance: 3038.913 timesteps/s, 55.801 Mparticle-step/s + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Move | 0.11792 | 0.1719 | 0.22427 | 12.4 | 52.24 +Coll | 0 | 0 | 0 | 0.0 | 0.00 +Sort | 0 | 0 | 0 | 0.0 | 0.00 +Comm | 0.013082 | 0.014985 | 0.016944 | 1.5 | 4.55 +Modify | 0.00018412 | 0.0046337 | 0.0091941 | 6.5 | 1.41 +Output | 0.010518 | 0.011383 | 0.013966 | 1.4 | 3.46 +MPI Sync| 0.033551 | 0.092822 | 0.15396 | 19.2 | 28.21 +Other | | 0.03334 | | | 10.13 + +Particle moves = 12288800 (12.3M) +Cells touched = 12477838 (12.5M) +Particle comms = 17683 (17.7K) +Boundary collides = 0 (0K) +Boundary exits = 36381 (36.4K) +SurfColl checks = 55089872 (55.1M) +SurfColl occurs = 56760 (56.8K) +Surf reactions = 56759 (56.8K) +Collide attempts = 0 (0K) +Collide occurs = 0 (0K) +Reactions = 0 (0K) +Particles stuck = 0 +Axisymm bad moves = 0 + +Particle-moves/CPUsec/proc: 9.33615e+06 +Particle-moves/step: 12288.8 +Cell-touches/particle/step: 1.01538 +Particle comm iterations/step: 1.814 +Particle fraction communicated: 0.00143895 +Particle fraction colliding with boundary: 0 +Particle fraction exiting boundary: 0.0029605 +Surface-checks/particle/step: 4.48293 +Surface-collisions/particle/step: 0.00461884 +Surf-reactions/particle/step: 0.00461876 +Collision-attempts/particle/step: 0 +Collisions/particle/step: 0 +Reactions/particle/step: 0 + +Surface reaction tallies: + id adsorb_test_gs_ps2 style adsorb #-of-reactions 14 + reaction all: 106080 + reaction O(g) --> O(s): 42345 + reaction O(g) --> CO(s): 13180 + reaction O(g) --> CO(g): 1233 + reaction C(g) --> C(b): 1 + reaction O(s) --> O(g): 24547 + reaction CO(s) --> CO(g): 14765 + reaction 2O(s) + C(b) --> CO2(g): 7772 + reaction O(s) + C(b) --> CO(s): 2111 + reaction C(b) --> C(g): 126 + +Particles: 4590.5 ave 6270 max 2950 min +Histogram: 2 0 0 0 0 0 0 0 0 2 +Cells: 2 ave 2 max 2 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +GhostCell: 6 ave 6 max 6 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +EmptyCell: 6 ave 6 max 6 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +Surfs: 12 ave 12 max 12 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +GhostSurf: 0 ave 0 max 0 min +Histogram: 4 0 0 0 0 0 0 0 0 0 diff --git a/examples/surf_react_adsorb/log.22Aug26.mpi_4.beam.surf.ps b/examples/surf_react_adsorb/log.22Aug26.mpi_4.beam.surf.ps new file mode 100644 index 000000000..722119cca Binary files /dev/null and b/examples/surf_react_adsorb/log.22Aug26.mpi_4.beam.surf.ps differ diff --git a/src/FFT/compute_fft_grid.cpp b/src/FFT/compute_fft_grid.cpp index 716f1095c..8fc88aaa8 100644 --- a/src/FFT/compute_fft_grid.cpp +++ b/src/FFT/compute_fft_grid.cpp @@ -945,7 +945,7 @@ void ComputeFFTGrid::print_FFT_info() { if (comm->me == 0) { char str[64]; - sprintf(str,"Using " SPARTA_FFT_PREC " precision " SPARTA_FFT_LIB " for FFTs\n"); + snprintf(str,sizeof(str),"Using " SPARTA_FFT_PREC " precision " SPARTA_FFT_LIB " for FFTs\n"); if (screen) fprintf(screen,"%s",str); if (logfile) fprintf(logfile,"%s",str); } diff --git a/src/KOKKOS/Install.sh b/src/KOKKOS/Install.sh index 1f66297d9..3d7c9a7a3 100644 --- a/src/KOKKOS/Install.sh +++ b/src/KOKKOS/Install.sh @@ -50,16 +50,28 @@ action compute_gas_collision_grid_kokkos.cpp action compute_gas_collision_grid_kokkos.h action compute_gas_reaction_grid_kokkos.cpp action compute_gas_reaction_grid_kokkos.h +action compute_gas_collision_tally_kokkos.cpp +action compute_gas_collision_tally_kokkos.h +action compute_gas_reaction_tally_kokkos.cpp +action compute_gas_reaction_tally_kokkos.h +action compute_surf_collision_tally_kokkos.cpp +action compute_surf_collision_tally_kokkos.h +action compute_surf_reaction_tally_kokkos.cpp +action compute_surf_reaction_tally_kokkos.h action compute_grid_kokkos.cpp action compute_grid_kokkos.h action compute_isurf_grid_kokkos.cpp action compute_isurf_grid_kokkos.h action compute_property_surf_kokkos.cpp action compute_property_surf_kokkos.h +action compute_react_boundary_kokkos.cpp +action compute_react_boundary_kokkos.h action compute_react_isurf_grid_kokkos.cpp action compute_react_isurf_grid_kokkos.h action compute_react_surf_kokkos.cpp action compute_react_surf_kokkos.h +action compute_reduce_kokkos.cpp +action compute_reduce_kokkos.h action compute_ke_particle_kokkos.cpp action compute_ke_particle_kokkos.h action compute_lambda_grid_kokkos.cpp @@ -169,9 +181,15 @@ action create_particles_kokkos.cpp action create_particles_kokkos.h action fix_emit_face_kokkos.cpp action fix_emit_face_kokkos.h +action fix_emit_face_file_kokkos.cpp +action fix_emit_face_file_kokkos.h action fix_emit_kokkos.h action fix_emit_surf_kokkos.cpp action fix_emit_surf_kokkos.h +action fix_field_grid_kokkos.cpp +action fix_field_grid_kokkos.h +action fix_field_particle_kokkos.cpp +action fix_field_particle_kokkos.h action fix_grid_check_kokkos.cpp action fix_grid_check_kokkos.h action read_surf_kokkos.cpp @@ -180,6 +198,11 @@ action region_block_kokkos.cpp action region_block_kokkos.h action region_cylinder_kokkos.cpp action region_cylinder_kokkos.h +action region_intersect_kokkos.cpp +action region_intersect_kokkos.h +action region_prim_kokkos.h +action region_union_kokkos.cpp +action region_union_kokkos.h action region_plane_kokkos.cpp action region_plane_kokkos.h action region_sphere_kokkos.cpp diff --git a/src/KOKKOS/collide_vss_kokkos.cpp b/src/KOKKOS/collide_vss_kokkos.cpp index 06ab9bab8..eaa2864e6 100644 --- a/src/KOKKOS/collide_vss_kokkos.cpp +++ b/src/KOKKOS/collide_vss_kokkos.cpp @@ -42,10 +42,42 @@ using namespace MathConst; #define VAL_1(X) X #define VAL_2(X) VAL_1(X), VAL_1(X) #define VAL_4(X) VAL_2(X), VAL_2(X) +// blit one active gas tally compute into its per-type device buffer +// same operation and rationale as KKCopy::copy() (kokkos_copy.h:71): the +// object is only read on device, through KOKKOS_INLINE_FUNCTION members, +// so its vtable pointer is never used and the View handles it carries stay +// alive in the original that update->glist_active holds + +#ifndef SPARTA_KOKKOS_FIXED_LISTS +namespace { + + template + void gas_buf_resize(DAT::tdual_char_1d &k, DAT::t_char_1d &d, int n) + { + const size_t need = (size_t) MAX(n,1) * sizeof(T); + if (k.view_device().extent(0) < need) { + k = DAT::tdual_char_1d("collide:gas_tally_models",need); + d = k.view_device(); + } + } + + template + void gas_buf_blit(DAT::tdual_char_1d &k, int slot, T *obj) + { + char *dst = k.view_host().data() + (size_t) slot*sizeof(T); + memcpy((void*) dst, (const void*) obj, sizeof(T)); + ((T *) dst)->copy = 1; + } -// the glist KKCopy arrays below are brace-initialized with VAL_4 (4 elements) -static_assert(KOKKOS_MAX_GLIST == 4, - "VAL_4 initializer lists assume KOKKOS_MAX_GLIST == 4"); + void gas_buf_sync(DAT::tdual_char_1d &k, DAT::t_char_1d &d) + { + if (k.view_device().extent(0) == 0) return; + k.modify_host(); + k.sync_device(); + d = k.view_device(); + } +} +#endif enum{NONE,DISCRETE,SMOOTH}; // several files enum{CONSTANT,VARIABLE}; @@ -54,9 +86,7 @@ enum{CONSTANT,VARIABLE}; #define DELTADELETE 1024 #define DELTAELECTRON 128 #define DELTACELLCOUNT 2 -#define MAXGROUP 16 // max # of collision groups for Kokkos group collisions -#define MAXLINE 1024 #define EPSZERO 1.0e-14 #define BIG 1.0e20 @@ -72,15 +102,22 @@ CollideVSSKokkos::CollideVSSKokkos(SPARTA *sparta, int narg, char **arg) : grid_kk_copy(sparta), react_kk_copy(sparta), react_qk_kk_copy(sparta), - react_tceqk_kk_copy(sparta), - glist_collision_copy{VAL_4(KKCopy(sparta))}, - glist_reaction_copy{VAL_4(KKCopy(sparta))}, - tmp_compute_gas_collision_kk(sparta), - tmp_compute_gas_reaction_kk(sparta) + react_tceqk_kk_copy(sparta) +#ifdef SPARTA_KOKKOS_FIXED_LISTS + , glist_collision_copy{VAL_4(KKCopy(sparta))} + , glist_coll_tally_copy{VAL_4(KKCopy(sparta))} + , glist_react_tally_copy{VAL_4(KKCopy(sparta))} + , glist_reaction_copy{VAL_4(KKCopy(sparta))} + , tmp_compute_gas_collision_kk(sparta) + , tmp_compute_gas_reaction_kk(sparta) + , tmp_compute_gas_coll_tally_kk(sparta) + , tmp_compute_gas_react_tally_kk(sparta) +#endif { kokkos_flag = 1; react_style = 0; nglist_collision = nglist_reaction = 0; + nglist_coll_tally = nglist_react_tally = 0; egroup = -1; // use 1D view for scalars to reduce GPU memory operations @@ -103,6 +140,7 @@ CollideVSSKokkos::CollideVSSKokkos(SPARTA *sparta, int narg, char **arg) : d_ndelete = Kokkos::subview(d_scalars,5); d_nlocal = Kokkos::subview(d_scalars,6); d_maxelectron = Kokkos::subview(d_scalars,7); + d_tally_overflow = Kokkos::subview(d_scalars,8); d_nattempt_one = Kokkos::subview(d_scalars_big,0); d_ncollide_one = Kokkos::subview(d_scalars_big,1); @@ -116,6 +154,7 @@ CollideVSSKokkos::CollideVSSKokkos(SPARTA *sparta, int narg, char **arg) : h_ndelete = Kokkos::subview(h_scalars,5); h_nlocal = Kokkos::subview(h_scalars,6); h_maxelectron = Kokkos::subview(h_scalars,7); + h_tally_overflow = Kokkos::subview(h_scalars,8); h_nattempt_one = Kokkos::subview(h_scalars_big,0); h_ncollide_one = Kokkos::subview(h_scalars_big,1); @@ -153,6 +192,13 @@ void CollideVSSKokkos::init() if (nparams != particle->nspecies) error->all(FLERR,"VSS parameters do not match current species"); + // CollideVSSKokkos::init() does not call the host base, so it carries its + // own copy of the host's restriction checks. These three mirror + // Collide::init() (collide.cpp:166-176) condition-for-condition and + // message-for-message: the ambipolar model has no near-neighbor or subcell + // implementation on the CPU either, so they are host restrictions being + // reproduced, not Kokkos limitations. + if (ambiflag && nearcp) error->all(FLERR,"Ambipolar collision model does not yet support " "near-neighbor collisions"); @@ -193,7 +239,7 @@ void CollideVSSKokkos::init() } if (flag) { char str[128]; - sprintf(str,"%d species do not define correct vibrational " + snprintf(str,sizeof(str),"%d species do not define correct vibrational " "modes for discrete model",flag); error->all(FLERR,str); } @@ -205,6 +251,9 @@ void CollideVSSKokkos::init() ngroups = mixture->ngroup; // must follow ngroups assignment above, as in Collide::init() + // mirrors collide.cpp:274-276; the host has no multigroup subcell algorithm + // either (Collide::subcell_alloc() refuses to allocate at collide.cpp:988), + // so there is no host result for a Kokkos version to reproduce if (subcellflag && ngroups > 1) error->all(FLERR,"Cannot yet use subcell collisions with " @@ -464,8 +513,9 @@ void CollideVSSKokkos::collisions() // variant for single group or multiple groups // partition active gas/gas tally computes by type into typed KKCopy lists - // each must be a supported Kokkos per-grid compute; call pre_gas_tally() - // the per-event gas/collision/tally and gas/reaction/tally are not supported + // each must be a Kokkos gas tally compute; call pre_gas_tally() on it + // covers the per-grid gas/collision/grid and gas/reaction/grid, and the + // per-event gas/collision/tally and gas/reaction/tally if (ngas_tally) setup_gas_tally(); @@ -503,22 +553,24 @@ void CollideVSSKokkos::collisions() } // multiple groups - // Kokkos currently supports only non-reacting, non-near-neighbor - // group collisions (with or without the ambipolar approximation) + // both the plain and the ambipolar group paths support reactions; the + // plain one also supports near-neighbor selection } else { - if (react) - error->all(FLERR,"Kokkos does not (yet) support reacting group collisions"); - if (nearcp) - error->all(FLERR,"Kokkos does not (yet) support near-neighbor group collisions"); + // unreachable: init() above already aborts this combination, matching the + // host. Kept as a belt-and-braces assert in case the dispatch is ever + // reached by another path + if (subcellflag) - error->all(FLERR,"Kokkos does not (yet) support subcell partners with " + error->all(FLERR,"Cannot yet use subcell collisions with " "multiple collision groups"); if (!ambiflag) { - if (!ngas_tally) { - collisions_group<0,0>(reduce); - } else if (ngas_tally) { - collisions_group<0,1>(reduce); + if (!nearcp) { + if (!ngas_tally) collisions_group<0,0>(reduce); + else collisions_group<0,1>(reduce); + } else { + if (!ngas_tally) collisions_group<1,0>(reduce); + else collisions_group<1,1>(reduce); } } else if (ambiflag) { if (!ngas_tally) { @@ -579,39 +631,86 @@ void CollideVSSKokkos::collisions() void CollideVSSKokkos::setup_gas_tally() { nglist_collision = nglist_reaction = 0; + nglist_coll_tally = nglist_react_tally = 0; // dispatch by dynamic_cast, not by style string, so a compute the user // typed with the explicit "/kk" suffix is still recognized + // count first: the buffers have to be sized before anything is blitted in + + for (int i = 0; i < ngas_tally; i++) { + Compute *c = update->glist_active[i]; + if (dynamic_cast(c)) nglist_collision++; + else if (dynamic_cast(c)) nglist_reaction++; + else if (dynamic_cast(c)) nglist_coll_tally++; + else if (dynamic_cast(c)) nglist_react_tally++; + else + error->all(FLERR,"Kokkos does not (yet) support this gas tally compute; " + "use a Kokkos-enabled gas tally compute (-sf kk)"); + } + +#ifdef SPARTA_KOKKOS_FIXED_LISTS + if (nglist_collision > KOKKOS_MAX_GLIST || nglist_reaction > KOKKOS_MAX_GLIST || + nglist_coll_tally > KOKKOS_MAX_GLIST || nglist_react_tally > KOKKOS_MAX_GLIST) + error->all(FLERR,"Kokkos supports at most KOKKOS_MAX_GLIST instances of each gas tally compute"); +#else + gas_buf_resize(k_glist_collision,d_glist_collision,nglist_collision); + gas_buf_resize(k_glist_reaction,d_glist_reaction,nglist_reaction); + gas_buf_resize(k_glist_coll_tally,d_glist_coll_tally,nglist_coll_tally); + gas_buf_resize(k_glist_react_tally,d_glist_react_tally,nglist_react_tally); +#endif + + int ncg = 0, nrg = 0, nct = 0, nrt = 0; for (int i = 0; i < ngas_tally; i++) { Compute *c = update->glist_active[i]; if (ComputeGasCollisionGridKokkos *ckk = dynamic_cast(c)) { - if (nglist_collision >= KOKKOS_MAX_GLIST) - error->all(FLERR,"Kokkos supports at most KOKKOS_MAX_GLIST instances of compute gas/collision/grid"); ckk->pre_gas_tally(); - glist_collision_copy[nglist_collision].copy(ckk); - nglist_collision++; +#ifdef SPARTA_KOKKOS_FIXED_LISTS + glist_collision_copy[ncg++].copy(ckk); +#else + gas_buf_blit(k_glist_collision,ncg++,ckk); +#endif } else if (ComputeGasReactionGridKokkos *ckk = dynamic_cast(c)) { - if (nglist_reaction >= KOKKOS_MAX_GLIST) - error->all(FLERR,"Kokkos supports at most KOKKOS_MAX_GLIST instances of compute gas/reaction/grid"); ckk->pre_gas_tally(); - glist_reaction_copy[nglist_reaction].copy(ckk); - nglist_reaction++; - } else { - error->all(FLERR,"Kokkos does not (yet) support this gas tally compute; " - "use a Kokkos-enabled gas tally compute (-sf kk)"); +#ifdef SPARTA_KOKKOS_FIXED_LISTS + glist_reaction_copy[nrg++].copy(ckk); +#else + gas_buf_blit(k_glist_reaction,nrg++,ckk); +#endif + } else if (ComputeGasCollisionTallyKokkos *ckk = + dynamic_cast(c)) { + ckk->pre_gas_tally(); + ckk->d_overflow = d_tally_overflow; +#ifdef SPARTA_KOKKOS_FIXED_LISTS + glist_coll_tally_copy[nct++].copy(ckk); +#else + gas_buf_blit(k_glist_coll_tally,nct++,ckk); +#endif + } else if (ComputeGasReactionTallyKokkos *ckk = + dynamic_cast(c)) { + ckk->pre_gas_tally(); + ckk->d_overflow = d_tally_overflow; +#ifdef SPARTA_KOKKOS_FIXED_LISTS + glist_react_tally_copy[nrt++].copy(ckk); +#else + gas_buf_blit(k_glist_react_tally,nrt++,ckk); +#endif } } - // fill unused slots of each typed copy list with the temporary - // to avoid the copy getting stale leading to an issue with view ref counting - - for (int i = nglist_collision; i < KOKKOS_MAX_GLIST; i++) - glist_collision_copy[i].copy(&tmp_compute_gas_collision_kk); - for (int i = nglist_reaction; i < KOKKOS_MAX_GLIST; i++) - glist_reaction_copy[i].copy(&tmp_compute_gas_reaction_kk); +#ifdef SPARTA_KOKKOS_FIXED_LISTS + for (int i = ncg; i < KOKKOS_MAX_GLIST; i++) glist_collision_copy[i].copy(&tmp_compute_gas_collision_kk); + for (int i = nrg; i < KOKKOS_MAX_GLIST; i++) glist_reaction_copy[i].copy(&tmp_compute_gas_reaction_kk); + for (int i = nct; i < KOKKOS_MAX_GLIST; i++) glist_coll_tally_copy[i].copy(&tmp_compute_gas_coll_tally_kk); + for (int i = nrt; i < KOKKOS_MAX_GLIST; i++) glist_react_tally_copy[i].copy(&tmp_compute_gas_react_tally_kk); +#else + gas_buf_sync(k_glist_collision,d_glist_collision); + gas_buf_sync(k_glist_reaction,d_glist_reaction); + gas_buf_sync(k_glist_coll_tally,d_glist_coll_tally); + gas_buf_sync(k_glist_react_tally,d_glist_react_tally); +#endif } /* ---------------------------------------------------------------------- @@ -627,6 +726,10 @@ void CollideVSSKokkos::finish_gas_tally() ckk->post_gas_tally(); else if (ComputeGasReactionGridKokkos *ckk = dynamic_cast(c)) ckk->post_gas_tally(); + else if (ComputeGasCollisionTallyKokkos *ckk = dynamic_cast(c)) + ckk->post_gas_tally(); + else if (ComputeGasReactionTallyKokkos *ckk = dynamic_cast(c)) + ckk->post_gas_tally(); } } @@ -669,12 +772,6 @@ template < int NEARCP, int GASTALLY > void CollideVSSKokkos::collisions_one(COLL grid_kk->sync(Device,CINFO_MASK); d_plist = grid_kk->d_plist; - if (react) { - ReactTCEKokkos* react_kk = (ReactTCEKokkos*) react; - if (!react_kk) - error->all(FLERR,"Must use TCE reactions with Kokkos"); - } - copymode = 1; if (NEARCP) { @@ -733,10 +830,25 @@ template < int NEARCP, int GASTALLY > void CollideVSSKokkos::collisions_one(COLL } } + // a per-event gas tally compute can force a retry of its own, and a retry + // re-runs the collision pass over the same particles. that is only sound + // if the particle list can be rolled back first, so the backup is not + // gated on react/retry when one of those computes is active + + const int tally_backup = (nglist_coll_tally || nglist_react_tally); + const int do_backup = + (react && sparta->kokkos->react_retry_flag) || tally_backup; + + if (tally_backup) rewind_gas_tally_computes(1); + while (h_retry()) { - if (react && sparta->kokkos->react_retry_flag) - backup(); + if (do_backup) backup(); + + // discard the rows an aborted attempt appended, including an attempt + // repeated for a reaction overflow rather than a tally overflow + + if (tally_backup) rewind_gas_tally_computes(0); h_retry() = 0; h_maxdelete() = maxdelete; @@ -745,6 +857,14 @@ template < int NEARCP, int GASTALLY > void CollideVSSKokkos::collisions_one(COLL h_ndelete() = 0; h_nlocal() = particle->nlocal; + // h_tally_overflow is not zeroed anywhere else on this path: the reaction + // retry branch below reads maxdelete/maxcellcount/nlocal back out of + // h_scalars, so unlike UpdateKokkos it cannot bulk-zero the array. A + // pass that raised both flags would otherwise push a stale 1 back to the + // device and trigger a spurious grow plus a wasted sweep next attempt + + h_tally_overflow() = 0; + Kokkos::deep_copy(d_scalars,h_scalars); Kokkos::deep_copy(d_scalars_big,h_scalars_big); @@ -783,9 +903,24 @@ template < int NEARCP, int GASTALLY > void CollideVSSKokkos::collisions_one(COLL Kokkos::deep_copy(h_scalars,d_scalars); Kokkos::deep_copy(h_scalars_big,d_scalars_big); + // a per-event gas tally compute ran out of room: grow it and repeat the + // pass. unlike a reaction overflow this needs no react/retry opt-in, + // and clear_gas_tally() below already discards the aborted pass + + if (h_tally_overflow() && !h_retry()) { + grow_gas_tally_computes(); + if (do_backup) restore(); + if (ngas_tally) clear_gas_tally(); + Kokkos::deep_copy(h_scalars,0); + Kokkos::deep_copy(h_scalars_big,0); + reduce = COLLIDE_REDUCE(); + h_retry() = 1; + continue; + } + if (h_retry()) { //printf("Retrying, reason %i %i %i !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n",h_maxdelete() > d_dellist.extent(0),h_maxcellcount() > d_plist.extent(1),h_part_grow()); - if (!sparta->kokkos->react_retry_flag) { + if (!do_backup) { error->one(FLERR,"Ran out of space in Kokkos collisions, increase react/extra" " or use react/retry"); } else @@ -969,9 +1104,13 @@ void CollideVSSKokkos::operator()(TagCollideCollisionsOne< NEARCP, GASTALLY, ATO if (GASTALLY) { for (int m = 0; m < nglist_collision; m++) - glist_collision_copy[m].obj.template gas_tally_kk(icell,reactflag,&iorig,&jorig,ipart,jpart,kpart); + CVK_GLIST_COLLISION(m).template gas_tally_kk(icell,reactflag,&iorig,&jorig,ipart,jpart,kpart); for (int m = 0; m < nglist_reaction; m++) - glist_reaction_copy[m].obj.template gas_tally_kk(icell,reactflag,&iorig,&jorig,ipart,jpart,kpart); + CVK_GLIST_REACTION(m).template gas_tally_kk(icell,reactflag,&iorig,&jorig,ipart,jpart,kpart); + for (int m = 0; m < nglist_coll_tally; m++) + CVK_GLIST_COLL_TALLY(m).template gas_tally_kk(icell,reactflag,&iorig,&jorig,ipart,jpart,kpart); + for (int m = 0; m < nglist_react_tally; m++) + CVK_GLIST_REACT_TALLY(m).template gas_tally_kk(icell,reactflag,&iorig,&jorig,ipart,jpart,kpart); } if (reactflag) { @@ -1070,12 +1209,6 @@ template < int DIM, int GASTALLY > void CollideVSSKokkos::collisions_one_subcell grid_kk->sync(Device,CINFO_MASK|CELL_MASK); d_plist = grid_kk->d_plist; - if (react) { - ReactTCEKokkos* react_kk = (ReactTCEKokkos*) react; - if (!react_kk) - error->all(FLERR,"Must use TCE reactions with Kokkos"); - } - copymode = 1; grow_subcell_views(nglocal,d_plist.extent(1)); @@ -1128,10 +1261,25 @@ template < int DIM, int GASTALLY > void CollideVSSKokkos::collisions_one_subcell } } + // a per-event gas tally compute can force a retry of its own, and a retry + // re-runs the collision pass over the same particles. that is only sound + // if the particle list can be rolled back first, so the backup is not + // gated on react/retry when one of those computes is active + + const int tally_backup = (nglist_coll_tally || nglist_react_tally); + const int do_backup = + (react && sparta->kokkos->react_retry_flag) || tally_backup; + + if (tally_backup) rewind_gas_tally_computes(1); + while (h_retry()) { - if (react && sparta->kokkos->react_retry_flag) - backup(); + if (do_backup) backup(); + + // discard the rows an aborted attempt appended, including an attempt + // repeated for a reaction overflow rather than a tally overflow + + if (tally_backup) rewind_gas_tally_computes(0); h_retry() = 0; h_maxdelete() = maxdelete; @@ -1140,6 +1288,14 @@ template < int DIM, int GASTALLY > void CollideVSSKokkos::collisions_one_subcell h_ndelete() = 0; h_nlocal() = particle->nlocal; + // h_tally_overflow is not zeroed anywhere else on this path: the reaction + // retry branch below reads maxdelete/maxcellcount/nlocal back out of + // h_scalars, so unlike UpdateKokkos it cannot bulk-zero the array. A + // pass that raised both flags would otherwise push a stale 1 back to the + // device and trigger a spurious grow plus a wasted sweep next attempt + + h_tally_overflow() = 0; + Kokkos::deep_copy(d_scalars,h_scalars); Kokkos::deep_copy(d_scalars_big,h_scalars_big); @@ -1178,8 +1334,23 @@ template < int DIM, int GASTALLY > void CollideVSSKokkos::collisions_one_subcell Kokkos::deep_copy(h_scalars,d_scalars); Kokkos::deep_copy(h_scalars_big,d_scalars_big); + // a per-event gas tally compute ran out of room: grow it and repeat the + // pass. unlike a reaction overflow this needs no react/retry opt-in, + // and clear_gas_tally() below already discards the aborted pass + + if (h_tally_overflow() && !h_retry()) { + grow_gas_tally_computes(); + if (do_backup) restore(); + if (ngas_tally) clear_gas_tally(); + Kokkos::deep_copy(h_scalars,0); + Kokkos::deep_copy(h_scalars_big,0); + reduce = COLLIDE_REDUCE(); + h_retry() = 1; + continue; + } + if (h_retry()) { - if (!sparta->kokkos->react_retry_flag) { + if (!do_backup) { error->one(FLERR,"Ran out of space in Kokkos collisions, increase react/extra" " or use react/retry"); } else @@ -1369,9 +1540,13 @@ void CollideVSSKokkos::operator()(TagCollideCollisionsOneSubcell< DIM, GASTALLY, if (GASTALLY) { for (int m = 0; m < nglist_collision; m++) - glist_collision_copy[m].obj.template gas_tally_kk(icell,reactflag,&iorig,&jorig,ipart,jpart,kpart); + CVK_GLIST_COLLISION(m).template gas_tally_kk(icell,reactflag,&iorig,&jorig,ipart,jpart,kpart); for (int m = 0; m < nglist_reaction; m++) - glist_reaction_copy[m].obj.template gas_tally_kk(icell,reactflag,&iorig,&jorig,ipart,jpart,kpart); + CVK_GLIST_REACTION(m).template gas_tally_kk(icell,reactflag,&iorig,&jorig,ipart,jpart,kpart); + for (int m = 0; m < nglist_coll_tally; m++) + CVK_GLIST_COLL_TALLY(m).template gas_tally_kk(icell,reactflag,&iorig,&jorig,ipart,jpart,kpart); + for (int m = 0; m < nglist_react_tally; m++) + CVK_GLIST_REACT_TALLY(m).template gas_tally_kk(icell,reactflag,&iorig,&jorig,ipart,jpart,kpart); } if (reactflag) { @@ -1711,19 +1886,31 @@ int CollideVSSKokkos::find_nn_subcell(rand_type &rand_gen, int i, int np, int ic return j; } +/* ---------------------------------------------------------------------- + resize the per-group lists to match the current d_plist capacity + a reaction can move every particle of a cell into one group, so each + group region must be able to hold the whole cell +------------------------------------------------------------------------- */ + +void CollideVSSKokkos::grow_group_lists() +{ + MemKK::realloc_kokkos(d_glist,"collide:glist",nglocal,ngroups,d_plist.extent(1)); + MemKK::realloc_kokkos(d_p2g,"collide:p2g",nglocal,d_plist.extent(1),2); + if (nearcp) { + MemKK::realloc_kokkos(d_nn_igroup,"collide:nn_igroup",nglocal,d_plist.extent(1)); + MemKK::realloc_kokkos(d_nn_jgroup,"collide:nn_jgroup",nglocal,d_plist.extent(1)); + } +} + /* ---------------------------------------------------------------------- NTC algorithm for multiple groups - Kokkos version supports only non-reacting, non-ambipolar, non-nearcp - collisions, so group membership is static within the timestep - and no particles are created or destroyed + supports reactions and near-neighbor selection; group membership changes + inside the kernel as reactions rebin, create and destroy particles ------------------------------------------------------------------------- */ template < int NEARCP, int GASTALLY > void CollideVSSKokkos::collisions_group(COLLIDE_REDUCE &reduce) { - if (ngroups > MAXGROUP) - error->all(FLERR,"Too many collision groups for Kokkos group collisions"); - // loop over cells I own this->sync(Device,ALL_MASK); @@ -1744,35 +1931,190 @@ void CollideVSSKokkos::collisions_group(COLLIDE_REDUCE &reduce) // d_glist holds plist indices laid out group-contiguous per cell // d_nattempt_pair holds the pre-computed attempt count per group pair + // one region per group, each able to hold the whole cell: a reaction can + // move every particle of a cell into the same group + if (int(d_glist.extent(0)) < nglocal || - int(d_glist.extent(1)) < int(d_plist.extent(1))) - MemKK::realloc_kokkos(d_glist,"collide:glist",nglocal,d_plist.extent(1)); + int(d_glist.extent(1)) < ngroups || + int(d_glist.extent(2)) < int(d_plist.extent(1))) { + MemKK::realloc_kokkos(d_glist,"collide:glist",nglocal,ngroups,d_plist.extent(1)); + MemKK::realloc_kokkos(d_p2g,"collide:p2g",nglocal,d_plist.extent(1),2); + } + if (nearcp && + (int(d_nn_igroup.extent(0)) < nglocal || + int(d_nn_igroup.extent(1)) < int(d_plist.extent(1)))) { + MemKK::realloc_kokkos(d_nn_igroup,"collide:nn_igroup",nglocal,d_plist.extent(1)); + MemKK::realloc_kokkos(d_nn_jgroup,"collide:nn_jgroup",nglocal,d_plist.extent(1)); + } if (int(d_nattempt_pair.extent(0)) < nglocal || int(d_nattempt_pair.extent(1)) < ngroups) MemKK::realloc_kokkos(d_nattempt_pair,"collide:nattempt_pair",nglocal,ngroups,ngroups); + // d_gcount holds the per-group particle counts the kernel used to keep in a + // per-thread stack array with a compile-time group cap. One row per cell, + // so the work item's icell is the row index: no token, no contention. + // Checked separately from d_glist because it does not scale with the + // d_plist capacity, so a reaction retry that grows d_plist leaves it alone. + + if (int(d_gcount.extent(0)) < nglocal || + int(d_gcount.extent(1)) < ngroups) + MemKK::realloc_kokkos(d_gcount,"collide:gcount",nglocal,ngroups); + copymode = 1; - // no particles are created or destroyed for non-reacting group collisions + // reactions can create or delete particles, so this needs the same + // grow-and-repeat loop collisions_one() uses: a Kokkos view cannot be + // grown inside a parallel loop, so the kernel raises d_retry and returns, + // the host reallocates, and the pass runs again - ndelete = 0; + h_retry() = 1; - h_error_flag() = 0; - Kokkos::deep_copy(d_scalars,h_scalars); - Kokkos::deep_copy(d_scalars_big,h_scalars_big); + if (react) { + double extra_factor = 1.0; + if (sparta->kokkos->react_retry_flag) + extra_factor = sparta->kokkos->react_extra; - grid_kk_copy.copy(grid_kk); + if (maxdelete*extra_factor > MAXSMALLINT) + error->one(FLERR,"Per-processor delete count is too big"); + int maxdelete_extra = maxdelete*extra_factor; + if (d_dellist.extent(0) < maxdelete_extra) { + memoryKK->destroy_kokkos(k_dellist,dellist); + memoryKK->create_kokkos(k_dellist,dellist,maxdelete_extra,"collide:dellist"); + d_dellist = k_dellist.view_device(); + } - if (sparta->kokkos->atomic_reduction) { - if (sparta->kokkos->need_atomics) - Kokkos::parallel_for(Kokkos::RangePolicy >(0,nglocal),*this); - else - Kokkos::parallel_for(Kokkos::RangePolicy >(0,nglocal),*this); - } else - Kokkos::parallel_reduce(Kokkos::RangePolicy >(0,nglocal),*this,reduce); + maxcellcount = particle_kk->get_maxcellcount(); + int maxcellcount_extra = maxcellcount*extra_factor; + if (d_plist.extent(1) < maxcellcount_extra) { + d_plist = {}; + Kokkos::resize(grid_kk->d_plist,nglocal,maxcellcount_extra); + d_plist = grid_kk->d_plist; + grow_group_lists(); + } + + bigint nlocal_extra = static_cast (particle->nlocal*extra_factor); + if (nlocal_extra > MAXSMALLINT) + error->one(FLERR,"Per-processor particle count is too big"); + if ((bigint) d_particles.extent(0) < nlocal_extra) { + particle->grow(nlocal_extra - particle->nlocal); + d_particles = particle_kk->k_particles.view_device(); + k_eiarray = particle_kk->k_eiarray; + } + } + + const int tally_backup = (nglist_coll_tally || nglist_react_tally); + const int do_backup = + (react && sparta->kokkos->react_retry_flag) || tally_backup; + + if (tally_backup) rewind_gas_tally_computes(1); + + while (h_retry()) { + + if (do_backup) backup(); + if (tally_backup) rewind_gas_tally_computes(0); + + h_retry() = 0; + h_maxdelete() = maxdelete; + h_maxcellcount() = maxcellcount; + h_part_grow() = 0; + h_ndelete() = 0; + h_nlocal() = particle->nlocal; + + // h_tally_overflow is not zeroed anywhere else on this path: the reaction + // retry branch below reads maxdelete/maxcellcount/nlocal back out of + // h_scalars, so unlike UpdateKokkos it cannot bulk-zero the array. A + // pass that raised both flags would otherwise push a stale 1 back to the + // device and trigger a spurious grow plus a wasted sweep next attempt + + h_tally_overflow() = 0; + h_error_flag() = 0; + + Kokkos::deep_copy(d_scalars,h_scalars); + Kokkos::deep_copy(d_scalars_big,h_scalars_big); + + grid_kk_copy.copy(grid_kk); + if (react) { + ReactQKKokkos* react_qk = dynamic_cast(react); + ReactTCEQKKokkos* react_tceqk = dynamic_cast(react); + if (react_tceqk) { + react_style = 2; + react_tceqk_kk_copy.copy(react_tceqk); + } else if (react_qk) { + react_style = 1; + react_qk_kk_copy.copy(react_qk); + } else { + react_style = 0; + react_kk_copy.copy((ReactTCEKokkos*) react); + } + } + + if (react) particle_kk->zero_custom_kokkos(); + + if (sparta->kokkos->atomic_reduction) { + if (sparta->kokkos->need_atomics) + Kokkos::parallel_for(Kokkos::RangePolicy >(0,nglocal),*this); + else + Kokkos::parallel_for(Kokkos::RangePolicy >(0,nglocal),*this); + } else + Kokkos::parallel_reduce(Kokkos::RangePolicy >(0,nglocal),*this,reduce); + + Kokkos::deep_copy(h_scalars,d_scalars); + Kokkos::deep_copy(h_scalars_big,d_scalars_big); + + if (h_tally_overflow() && !h_retry()) { + grow_gas_tally_computes(); + if (do_backup) restore(); + if (ngas_tally) clear_gas_tally(); + Kokkos::deep_copy(h_scalars,0); + Kokkos::deep_copy(h_scalars_big,0); + reduce = COLLIDE_REDUCE(); + h_retry() = 1; + continue; + } + + if (h_retry()) { + if (!do_backup) { + error->one(FLERR,"Ran out of space in Kokkos collisions, increase react/extra" + " or use react/retry"); + } else + restore(); + + if (ngas_tally) clear_gas_tally(); + + reduce = COLLIDE_REDUCE(); - Kokkos::deep_copy(h_scalars,d_scalars); - Kokkos::deep_copy(h_scalars_big,d_scalars_big); + maxdelete = h_maxdelete(); + if (d_dellist.extent(0) < maxdelete) { + memoryKK->destroy_kokkos(k_dellist,dellist); + memoryKK->grow_kokkos(k_dellist,dellist,maxdelete,"collide:dellist"); + d_dellist = k_dellist.view_device(); + } + + maxcellcount = h_maxcellcount(); + particle_kk->set_maxcellcount(maxcellcount); + if (d_plist.extent(1) < maxcellcount) { + d_plist = {}; + Kokkos::resize(grid_kk->d_plist,nglocal,maxcellcount); + d_plist = grid_kk->d_plist; + grow_group_lists(); + } + + auto nlocal_new = h_nlocal(); + if (d_particles.extent(0) < nlocal_new) { + particle->grow(nlocal_new - particle->nlocal); + d_particles = particle_kk->k_particles.view_device(); + k_eiarray = particle_kk->k_eiarray; + } + } + } + + ndelete = h_ndelete(); + + // publish the particles the reactions created: the kernel appended them to + // the device list and counted them in d_nlocal, but until nlocal is + // carried back the host cannot see them + + particle->nlocal = h_nlocal(); copymode = 0; @@ -1784,6 +2126,14 @@ void CollideVSSKokkos::collisions_group(COLLIDE_REDUCE &reduce) if (vibstyle == DISCRETE) particle_kk->modify(Device,CUSTOM_MASK); d_particles = t_particle_1d(); // destroy reference to reduce memory use + + // d_nn_igroup/d_nn_jgroup are owned by this class, not borrowed from grid + // or particle, so they are not released here. Freeing them made the + // guard above reallocate two nglocal x maxcellcount int arrays on every + // timestep of every nearcp multigroup run; they are sized by that guard + // and reused instead. (d_particles and d_plist are references into + // other classes' allocations, so those are still dropped.) + d_plist = {}; } @@ -1797,6 +2147,12 @@ void CollideVSSKokkos::operator()(TagCollideCollisionsGroup< NEARCP, GASTALLY, A template < int NEARCP, int GASTALLY, int ATOMIC_REDUCTION > KOKKOS_INLINE_FUNCTION void CollideVSSKokkos::operator()(TagCollideCollisionsGroup< NEARCP, GASTALLY, ATOMIC_REDUCTION >, const int &icell, COLLIDE_REDUCE &reduce) const { + // once any cell has raised d_retry the whole pass is going to be rolled + // back and re-run, so the remaining cells should not do a full reacting + // pass whose results are only going to be discarded. the other four + // collision kernels all start this way + + if (d_retry()) return; int np = grid_kk_copy.obj.d_cellcount[icell]; if (np <= 1) return; @@ -1805,30 +2161,14 @@ void CollideVSSKokkos::operator()(TagCollideCollisionsGroup< NEARCP, GASTALLY, A if (volume == 0.0) d_error_flag() = 1; // build per-group particle lists for this cell - // gcount[g] = # of particles in group g - // gstart[g] = offset of group g within d_glist(icell,*) - // d_glist(icell,k) = plist index of kth particle, laid out group-contiguous - // in the same per-group order as the non-Kokkos version - - int gcount[MAXGROUP]; - int gstart[MAXGROUP]; - int gcursor[MAXGROUP]; + // d_gcount(icell,g) = # of particles in group g + // d_glist(icell,g,k) = plist index of the kth particle of group g + // built with addgroup_kk in plist order, as the non-Kokkos version does - for (int g = 0; g < ngroups; g++) gcount[g] = 0; - for (int n = 0; n < np; n++) { - const int isp = d_particles[d_plist(icell,n)].ispecies; - gcount[d_species2group[isp]]++; - } - int offset = 0; - for (int g = 0; g < ngroups; g++) { - gstart[g] = offset; - gcursor[g] = offset; - offset += gcount[g]; - } + for (int g = 0; g < ngroups; g++) d_gcount(icell,g) = 0; for (int n = 0; n < np; n++) { const int isp = d_particles[d_plist(icell,n)].ispecies; - const int g = d_species2group[isp]; - d_glist(icell,gcursor[g]++) = n; + addgroup_kk(icell,d_species2group[isp],n); } struct State precoln; // state before collision @@ -1843,7 +2183,8 @@ void CollideVSSKokkos::operator()(TagCollideCollisionsGroup< NEARCP, GASTALLY, A for (int ig = 0; ig < ngroups; ig++) for (int jg = ig; jg < ngroups; jg++) { const double attempt = - attempt_collision_kokkos(icell,ig,jg,gcount[ig],gcount[jg],volume,rand_gen); + attempt_collision_kokkos(icell,ig,jg,d_gcount(icell,ig), + d_gcount(icell,jg),volume,rand_gen); const int nattempt = static_cast (attempt); d_nattempt_pair(icell,ig,jg) = nattempt; if (nattempt) { @@ -1864,27 +2205,70 @@ void CollideVSSKokkos::operator()(TagCollideCollisionsGroup< NEARCP, GASTALLY, A for (int jg = ig; jg < ngroups; jg++) { const int nattempt = d_nattempt_pair(icell,ig,jg); if (!nattempt) continue; - const int ni = gcount[ig]; - const int nj = gcount[jg]; - if (ni == 0 || nj == 0) continue; - if (ig == jg && ni == 1) continue; + if (d_gcount(icell,ig) == 0 || d_gcount(icell,jg) == 0) continue; + if (ig == jg && d_gcount(icell,ig) == 1) continue; + + // near-neighbor bookkeeping is per group pair and starts cleared, + // as Collide::collisions_group() does via set_nn_group() + + if (NEARCP) { + const int nclear_i = d_gcount(icell,ig); + for (int k = 0; k < nclear_i; k++) d_nn_igroup(icell,k) = 0; + if (ig != jg) { + const int nclear_j = d_gcount(icell,jg); + for (int k = 0; k < nclear_j; k++) d_nn_jgroup(icell,k) = 0; + } + } for (int iattempt = 0; iattempt < nattempt; iattempt++) { + const int ni = d_gcount(icell,ig); + const int nj = d_gcount(icell,jg); + int i = ni * rand_gen.drand(); - int j = nj * rand_gen.drand(); - if (ig == jg) - while (i == j) j = nj * rand_gen.drand(); + int j; + if (NEARCP) j = find_nn_group(rand_gen,icell,i,ig,jg,ni,nj); + else { + j = nj * rand_gen.drand(); + if (ig == jg) + while (i == j) j = nj * rand_gen.drand(); + } - Particle::OnePart* ipart = &d_particles[d_plist(icell,d_glist(icell,gstart[ig]+i))]; - Particle::OnePart* jpart = &d_particles[d_plist(icell,d_glist(icell,gstart[jg]+j))]; + const int ii = d_glist(icell,ig,i); + const int jj = d_glist(icell,jg,j); + + Particle::OnePart* ipart = &d_particles[d_plist(icell,ii)]; + Particle::OnePart* jpart = &d_particles[d_plist(icell,jj)]; // test if collision actually occurs if (!test_collision_kokkos(icell,ig,jg,ipart,jpart,precoln,rand_gen)) continue; - // perform collision - // non-reacting: no chemistry, no 3rd particle, no create/delete - // if GASTALLY: save iorig/jorig for tally (tally hook deferred) + if (NEARCP) { + d_nn_igroup(icell,i) = j+1; + if (ig == jg) d_nn_igroup(icell,j) = i+1; + else d_nn_jgroup(icell,j) = i+1; + } + + // if recombination is possible for this IJ pair, pick a 3rd particle + // and set the cell number density, unless the boost factor turns it + // off or there is no 3rd particle + + Particle::OnePart* recomb_part3 = NULL; + int recomb_species = -1; + double recomb_density = 0.0; + if (recombflag && d_recomb_ijflag(ipart->ispecies,jpart->ispecies)) { + if (rand_gen.drand() > recomb_boost_inverse) + recomb_species = -1; + else if (np <= 2) + recomb_species = -1; + else { + int k = np * rand_gen.drand(); + while (k == ii || k == jj) k = np * rand_gen.drand(); + recomb_part3 = &d_particles[d_plist(icell,k)]; + recomb_species = recomb_part3->ispecies; + recomb_density = np * fnum / volume; + } + } Particle::OnePart iorig,jorig; if (GASTALLY) { @@ -1893,14 +2277,12 @@ void CollideVSSKokkos::operator()(TagCollideCollisionsGroup< NEARCP, GASTALLY, A } Particle::OnePart* kpart = NULL; - Particle::OnePart* recomb_part3 = NULL; - int recomb_species = -1; - double recomb_density = 0.0; int index_kpart = 0; setup_collision_kokkos(ipart,jpart,precoln,postcoln); - const int reactflag = perform_collision_kokkos(ipart,jpart,kpart,precoln,postcoln,rand_gen, - recomb_part3,recomb_species,recomb_density,index_kpart); + const int reactflag = + perform_collision_kokkos(ipart,jpart,kpart,precoln,postcoln,rand_gen, + recomb_part3,recomb_species,recomb_density,index_kpart); if (ATOMIC_REDUCTION == 1) Kokkos::atomic_inc(&d_ncollide_one()); @@ -1911,9 +2293,114 @@ void CollideVSSKokkos::operator()(TagCollideCollisionsGroup< NEARCP, GASTALLY, A if (GASTALLY) { for (int m = 0; m < nglist_collision; m++) - glist_collision_copy[m].obj.template gas_tally_kk(icell,reactflag,&iorig,&jorig,ipart,jpart,kpart); + CVK_GLIST_COLLISION(m).template gas_tally_kk(icell,reactflag,&iorig,&jorig,ipart,jpart,kpart); for (int m = 0; m < nglist_reaction; m++) - glist_reaction_copy[m].obj.template gas_tally_kk(icell,reactflag,&iorig,&jorig,ipart,jpart,kpart); + CVK_GLIST_REACTION(m).template gas_tally_kk(icell,reactflag,&iorig,&jorig,ipart,jpart,kpart); + for (int m = 0; m < nglist_coll_tally; m++) + CVK_GLIST_COLL_TALLY(m).template gas_tally_kk(icell,reactflag,&iorig,&jorig,ipart,jpart,kpart); + for (int m = 0; m < nglist_react_tally; m++) + CVK_GLIST_REACT_TALLY(m).template gas_tally_kk(icell,reactflag,&iorig,&jorig,ipart,jpart,kpart); + } + + if (reactflag) { + if (ATOMIC_REDUCTION == 1) + Kokkos::atomic_inc(&d_nreact_one()); + else if (ATOMIC_REDUCTION == 0) + d_nreact_one()++; + else + reduce.nreact_one++; + } else continue; + + // ipart may now belong to a different group + + int newgroup = d_species2group[ipart->ispecies]; + if (newgroup != ig) { + addgroup_kk(icell,newgroup,ii); + delgroup_kk(icell,ig,i); + // needed if jg == ig and delgroup moved the J particle + if (jg == ig && j == d_gcount(icell,ig)) j = i; + } + + // jpart may now belong to a different group, or have been destroyed + + if (jpart) { + newgroup = d_species2group[jpart->ispecies]; + if (newgroup != jg) { + addgroup_kk(icell,newgroup,jj); + delgroup_kk(icell,jg,j); + } + + } else { + const int ndelete = Kokkos::atomic_fetch_add(&d_ndelete(),1); + if (ndelete < d_dellist.extent(0)) { + d_dellist(ndelete) = d_plist(icell,jj); + } else { + d_retry() = 1; + d_maxdelete() += DELTADELETE; + rand_pool.free_state(rand_gen); + return; + } + + delgroup_kk(icell,jg,j); + + // swap-remove jj from plist and repair the moved entry's group entry + // through the reverse map, as Collide does with p2g + + np--; + d_plist(icell,jj) = d_plist(icell,np); + if (jj < np) { + const int mg = d_p2g(icell,np,0); + const int mk = d_p2g(icell,np,1); + d_glist(icell,mg,mk) = jj; + d_p2g(icell,jj,0) = mg; + d_p2g(icell,jj,1) = mk; + } + + if (NEARCP) { + if (ig == jg) d_nn_igroup(icell,j) = d_nn_igroup(icell,d_gcount(icell,jg)); + else d_nn_jgroup(icell,j) = d_nn_jgroup(icell,d_gcount(icell,jg)); + } + } + + // if kpart was created, append it to plist and to its group + + if (kpart) { + newgroup = d_species2group[kpart->ispecies]; + + if (np < d_plist.extent(1)) { + // the host clears the new particle's slot in BOTH nn arrays of + // the current pair (collide.cpp:1379-1390); when ig == jg the + // two alias, so one write covers it + + if (NEARCP) { + if (newgroup == ig || newgroup == jg) { + const int n = d_gcount(icell,newgroup); + d_nn_igroup(icell,n) = 0; + if (ig != jg) d_nn_jgroup(icell,n) = 0; + } + } + d_plist(icell,np) = index_kpart; + addgroup_kk(icell,newgroup,np); + np++; + } else { + d_retry() = 1; + d_maxcellcount() += DELTACELLCOUNT; + rand_pool.free_state(rand_gen); + return; + } + } + + // stop attempting if either group has become too small + + const int nig = d_gcount(icell,ig); + if (nig <= 1) { + if (nig == 0) break; + if (ig == jg) break; + } + const int njg = d_gcount(icell,jg); + if (njg <= 1) { + if (njg == 0) break; + if (ig == jg) break; } } } @@ -1923,76 +2410,273 @@ void CollideVSSKokkos::operator()(TagCollideCollisionsGroup< NEARCP, GASTALLY, A /* ---------------------------------------------------------------------- NTC algorithm for multiple groups with ambipolar approximation - Kokkos version supports only the non-reacting case, so group membership - and the electron list are static within the timestep and no particles - are created or destroyed + supports reactions: group membership, the electron list and the cell + particle list all change inside the kernel as reactions rebin, create + and destroy particles + ports Collide::collisions_group_ambipolar() (collide.cpp:1727-2135); the + order in which the lists are mutated is load bearing, because rebinning + changes which index a later random draw lands on, so every add/del is + placed exactly where the host places it ------------------------------------------------------------------------- */ template < int GASTALLY > void CollideVSSKokkos::collisions_group_ambipolar(COLLIDE_REDUCE &reduce) { - if (ngroups > MAXGROUP) - error->all(FLERR,"Too many collision groups for Kokkos group collisions"); - // ambipolar vectors this->sync(Device,ALL_MASK); - ParticleKokkos* particle_kk = (ParticleKokkos*) particle; - particle_kk->sync(Device,PARTICLE_MASK|SPECIES_MASK|CUSTOM_MASK); - d_particles = particle_kk->k_particles.view_device(); - d_species = particle_kk->k_species.view_device(); - d_ewhich = particle_kk->k_ewhich.view_device(); - auto h_ewhich = particle_kk->k_ewhich.view_host(); - k_eivec = particle_kk->k_eivec; - k_eiarray = particle_kk->k_eiarray; - k_edarray = particle_kk->k_edarray; - d_ionambi = k_eivec.view_host()[h_ewhich[index_ionambi]].k_view.view_device(); - d_velambi = k_edarray.view_host()[h_ewhich[index_velambi]].k_view.view_device(); + ParticleKokkos* particle_kk = (ParticleKokkos*) particle; + particle_kk->sync(Device,PARTICLE_MASK|SPECIES_MASK|CUSTOM_MASK); + d_particles = particle_kk->k_particles.view_device(); + d_species = particle_kk->k_species.view_device(); + d_ewhich = particle_kk->k_ewhich.view_device(); + auto h_ewhich = particle_kk->k_ewhich.view_host(); + k_eivec = particle_kk->k_eivec; + k_eiarray = particle_kk->k_eiarray; + k_edarray = particle_kk->k_edarray; + d_ionambi = k_eivec.view_host()[h_ewhich[index_ionambi]].k_view.view_device(); + d_velambi = k_edarray.view_host()[h_ewhich[index_velambi]].k_view.view_device(); + + GridKokkos* grid_kk = (GridKokkos*) grid; + grid_kk->sync(Device,CINFO_MASK); + d_plist = grid_kk->d_plist; + + // allocate per-cell group scratch arrays (see collisions_group) + // one region per group, each able to hold the whole cell: a reaction can + // move every particle of a cell into the same group + + if (int(d_glist.extent(0)) < nglocal || + int(d_glist.extent(1)) < ngroups || + int(d_glist.extent(2)) < int(d_plist.extent(1))) + grow_group_lists(); + if (int(d_nattempt_pair.extent(0)) < nglocal || + int(d_nattempt_pair.extent(1)) < ngroups) + MemKK::realloc_kokkos(d_nattempt_pair,"collide:nattempt_pair",nglocal,ngroups,ngroups); + + // per-cell group counters, formerly per-thread stack arrays with a + // compile-time group cap. shared with collisions_group() + + if (int(d_gcount.extent(0)) < nglocal || + int(d_gcount.extent(1)) < ngroups) + MemKK::realloc_kokkos(d_gcount,"collide:gcount",nglocal,ngroups); + + copymode = 1; + + // reactions can create or delete particles and electrons, so this needs the + // same grow-and-repeat loop the other reacting paths use: a Kokkos view + // cannot be grown inside a parallel loop, so the kernel raises d_retry and + // returns, the host reallocates, and the pass runs again + + h_retry() = 1; + + // the elist of split-off ambipolar electrons must be allocated whether or + // not reactions are defined: ambipolar collisions create a temporary + // electron for every ambipolar ion on every timestep. only the extra + // sizing for reaction-created particles and deletions is react-specific + + double extra_factor = 1.0; + if (react && sparta->kokkos->react_retry_flag) + extra_factor = sparta->kokkos->react_extra; + + maxcellcount = particle_kk->get_maxcellcount(); + + int maxelectron_extra = maxcellcount*extra_factor; + if (int(d_elist.extent(0)) < nglocal || int(d_elist.extent(1)) < maxelectron_extra) { + d_elist = t_particle_2d(); // reduce memory use by deallocating first + d_elist = t_particle_2d(Kokkos::view_alloc("collide:elist",Kokkos::WithoutInitializing),nglocal,maxelectron_extra); + } + + if (react) { + // form the product in double and check it before it becomes an int, + // dellist is indexed by an int + + if (maxdelete*extra_factor > MAXSMALLINT) + error->one(FLERR,"Per-processor delete count is too big"); + int maxdelete_extra = maxdelete*extra_factor; + if (d_dellist.extent(0) < maxdelete_extra) { + memoryKK->destroy_kokkos(k_dellist,dellist); + memoryKK->grow_kokkos(k_dellist,dellist,maxdelete_extra,"collide:dellist"); + d_dellist = k_dellist.view_device(); + } + + int maxcellcount_extra = maxcellcount*extra_factor; + if (d_plist.extent(1) < maxcellcount_extra) { + d_plist = {}; + Kokkos::resize(grid_kk->d_plist,nglocal,maxcellcount_extra); + d_plist = grid_kk->d_plist; + grow_group_lists(); + } + + bigint nlocal_extra = static_cast (particle->nlocal*extra_factor); + if (nlocal_extra > MAXSMALLINT) + error->one(FLERR,"Per-processor particle count is too big"); + if ((bigint) d_particles.extent(0) < nlocal_extra) { + particle->grow(nlocal_extra - particle->nlocal); + particle_kk->sync(Device,PARTICLE_MASK|SPECIES_MASK|CUSTOM_MASK); + d_particles = particle_kk->k_particles.view_device(); + auto h_ewhich = particle_kk->k_ewhich.view_host(); + k_eivec = particle_kk->k_eivec; + k_eiarray = particle_kk->k_eiarray; + k_edarray = particle_kk->k_edarray; + d_ionambi = k_eivec.view_host()[h_ewhich[index_ionambi]].k_view.view_device(); + d_velambi = k_edarray.view_host()[h_ewhich[index_velambi]].k_view.view_device(); + } + } + + // a per-event gas tally compute can force a retry of its own, and a retry + // re-runs the collision pass over the same particles. that is only sound + // if the particle list can be rolled back first, so the backup is not + // gated on react/retry when one of those computes is active + + const int tally_backup = (nglist_coll_tally || nglist_react_tally); + const int do_backup = + (react && sparta->kokkos->react_retry_flag) || tally_backup; + + if (tally_backup) rewind_gas_tally_computes(1); + + while (h_retry()) { + + if (do_backup) backup(); + + // discard the rows an aborted attempt appended, including an attempt + // repeated for a reaction overflow rather than a tally overflow + + if (tally_backup) rewind_gas_tally_computes(0); + + h_retry() = 0; + // seed from the allocation, not from the Collide member: maxelectron + // starts at 0 and the kernel only overflows against d_elist.extent(1), + // so seeding it with 0 makes each retry bump the request by + // DELTACELLCOUNT from zero and re-run the whole pass until it finally + // exceeds the extent -- dozens of wasted sweeps before the realloc + + maxelectron = d_elist.extent(1); + h_maxelectron() = maxelectron; + h_maxdelete() = maxdelete; + h_maxcellcount() = maxcellcount; + h_part_grow() = 0; + h_ndelete() = 0; + h_nlocal() = particle->nlocal; + + // h_tally_overflow is not zeroed anywhere else on this path: the reaction + // retry branch below reads maxdelete/maxcellcount/nlocal back out of + // h_scalars, so unlike UpdateKokkos it cannot bulk-zero the array. A + // pass that raised both flags would otherwise push a stale 1 back to the + // device and trigger a spurious grow plus a wasted sweep next attempt + + h_tally_overflow() = 0; + h_error_flag() = 0; + + Kokkos::deep_copy(d_scalars,h_scalars); + Kokkos::deep_copy(d_scalars_big,h_scalars_big); + + grid_kk_copy.copy(grid_kk); + if (react) { + ReactQKKokkos* react_qk = dynamic_cast(react); + ReactTCEQKKokkos* react_tceqk = dynamic_cast(react); + if (react_tceqk) { + react_style = 2; + react_tceqk_kk_copy.copy(react_tceqk); + } else if (react_qk) { + react_style = 1; + react_qk_kk_copy.copy(react_qk); + } else { + react_style = 0; + react_kk_copy.copy((ReactTCEKokkos*) react); + } + } + + // zero the custom attributes of the slots a reaction can fill + // must precede the kernel, not follow it: ambi_reset_kokkos() sets the + // ion flag of the third product the reaction just created, and + // EEXCHANGE_ReactingEDisposal() sets its vibrational mode levels + + if (react) particle_kk->zero_custom_kokkos(); + + if (sparta->kokkos->atomic_reduction) { + if (sparta->kokkos->need_atomics) + Kokkos::parallel_for(Kokkos::RangePolicy >(0,nglocal),*this); + else + Kokkos::parallel_for(Kokkos::RangePolicy >(0,nglocal),*this); + } else + Kokkos::parallel_reduce(Kokkos::RangePolicy >(0,nglocal),*this,reduce); + + Kokkos::deep_copy(h_scalars,d_scalars); + Kokkos::deep_copy(h_scalars_big,d_scalars_big); + + // a per-event gas tally compute ran out of room: grow it and repeat the + // pass. unlike a reaction overflow this needs no react/retry opt-in, + // and clear_gas_tally() below already discards the aborted pass - GridKokkos* grid_kk = (GridKokkos*) grid; - grid_kk->sync(Device,CINFO_MASK); - d_plist = grid_kk->d_plist; + if (h_tally_overflow() && !h_retry()) { + grow_gas_tally_computes(); + if (do_backup) restore(); + if (ngas_tally) clear_gas_tally(); + Kokkos::deep_copy(h_scalars,0); + Kokkos::deep_copy(h_scalars_big,0); + reduce = COLLIDE_REDUCE(); + h_retry() = 1; + continue; + } - // allocate per-cell group scratch arrays (see collisions_group) + if (h_retry()) { + if (!do_backup) { + error->one(FLERR,"Ran out of space in Kokkos collisions, increase react/extra" + " or use react/retry"); + } else + restore(); - if (int(d_glist.extent(0)) < nglocal || - int(d_glist.extent(1)) < int(d_plist.extent(1))) - MemKK::realloc_kokkos(d_glist,"collide:glist",nglocal,d_plist.extent(1)); - if (int(d_nattempt_pair.extent(0)) < nglocal || - int(d_nattempt_pair.extent(1)) < ngroups) - MemKK::realloc_kokkos(d_nattempt_pair,"collide:nattempt_pair",nglocal,ngroups,ngroups); + // undo gas tally events from the aborted pass before the kernel re-runs - // per-cell electron list; non-reacting so nelectron <= cell particle count + if (ngas_tally) clear_gas_tally(); - maxcellcount = particle_kk->get_maxcellcount(); - if (int(d_elist.extent(0)) < nglocal || int(d_elist.extent(1)) < maxcellcount) { - d_elist = t_particle_2d(); // reduce memory use by deallocating first - d_elist = t_particle_2d(Kokkos::view_alloc("collide:elist",Kokkos::WithoutInitializing),nglocal,maxcellcount); - } + reduce = COLLIDE_REDUCE(); - copymode = 1; + maxelectron = h_maxelectron(); + if (int(d_elist.extent(1)) < maxelectron) { + d_elist = t_particle_2d(); // reduce memory use by deallocating first + d_elist = t_particle_2d(Kokkos::view_alloc("collide:elist",Kokkos::WithoutInitializing),nglocal,maxelectron); + } - // no particles are created or destroyed for non-reacting group collisions + maxdelete = h_maxdelete(); + if (d_dellist.extent(0) < maxdelete) { + memoryKK->destroy_kokkos(k_dellist,dellist); + memoryKK->grow_kokkos(k_dellist,dellist,maxdelete,"collide:dellist"); + d_dellist = k_dellist.view_device(); + } - ndelete = 0; + maxcellcount = h_maxcellcount(); + particle_kk->set_maxcellcount(maxcellcount); + if (d_plist.extent(1) < maxcellcount) { + d_plist = {}; + Kokkos::resize(grid_kk->d_plist,nglocal,maxcellcount); + d_plist = grid_kk->d_plist; + grow_group_lists(); + } - h_error_flag() = 0; - Kokkos::deep_copy(d_scalars,h_scalars); - Kokkos::deep_copy(d_scalars_big,h_scalars_big); + auto nlocal_new = h_nlocal(); + if (d_particles.extent(0) < nlocal_new) { + particle->grow(nlocal_new - particle->nlocal); + particle_kk->sync(Device,PARTICLE_MASK|SPECIES_MASK|CUSTOM_MASK); + d_particles = particle_kk->k_particles.view_device(); + auto h_ewhich = particle_kk->k_ewhich.view_host(); + k_eivec = particle_kk->k_eivec; + k_eiarray = particle_kk->k_eiarray; + k_edarray = particle_kk->k_edarray; + d_ionambi = k_eivec.view_host()[h_ewhich[index_ionambi]].k_view.view_device(); + d_velambi = k_edarray.view_host()[h_ewhich[index_velambi]].k_view.view_device(); + } + } + } - grid_kk_copy.copy(grid_kk); + ndelete = h_ndelete(); - if (sparta->kokkos->atomic_reduction) { - if (sparta->kokkos->need_atomics) - Kokkos::parallel_for(Kokkos::RangePolicy >(0,nglocal),*this); - else - Kokkos::parallel_for(Kokkos::RangePolicy >(0,nglocal),*this); - } else - Kokkos::parallel_reduce(Kokkos::RangePolicy >(0,nglocal),*this,reduce); + // publish the particles the reactions created: the kernel appended them to + // the device list and counted them in d_nlocal, but until nlocal is + // carried back the host cannot see them - Kokkos::deep_copy(h_scalars,d_scalars); - Kokkos::deep_copy(h_scalars_big,d_scalars_big); + particle->nlocal = h_nlocal(); copymode = 0; @@ -2018,6 +2702,7 @@ void CollideVSSKokkos::operator()(TagCollideCollisionsGroupAmbipolar< GASTALLY, template < int GASTALLY, int ATOMIC_REDUCTION > KOKKOS_INLINE_FUNCTION void CollideVSSKokkos::operator()(TagCollideCollisionsGroupAmbipolar< GASTALLY, ATOMIC_REDUCTION >, const int &icell, COLLIDE_REDUCE &reduce) const { + if (d_retry()) return; int np = grid_kk_copy.obj.d_cellcount[icell]; if (np <= 1) return; @@ -2025,49 +2710,38 @@ void CollideVSSKokkos::operator()(TagCollideCollisionsGroupAmbipolar< GASTALLY, const double volume = grid_kk_copy.obj.k_cinfo.view_device()[icell].volume / grid_kk_copy.obj.k_cinfo.view_device()[icell].weight; if (volume == 0.0) d_error_flag() = 1; - // build per-group particle lists for this cell, plus the electron list - // gcount[g] = particle count in group g, with the electron count for egroup - // gstart[g] = offset of group g's real particles within d_glist(icell,*) - // (the electron group egroup has no real particles, so it adds no entries) - // electrons (one per ambipolar ion) are created in d_elist in plist order - - int gcount[MAXGROUP]; - int gstart[MAXGROUP]; - int gcursor[MAXGROUP]; - - for (int g = 0; g < ngroups; g++) gcount[g] = 0; + // build the per-group particle lists for this cell and the electron list, + // in one pass over plist, exactly as collide.cpp:1792-1824 does + // d_glist(icell,g,k) = plist index of the kth particle of group g + // d_p2g(icell,n,*) = reverse map, group and slot within it, for plist n + // d_gcount(icell,g) = particle count in group g + // d_elist(icell,e) = the eth ionized electron, split off from its ion + // + // the electron group egroup is the one exception: its d_gcount is + // maintained but its d_glist row is not written, because the host's + // glist[egroup][k] is always k (electrons are appended in order and + // removed by swapping the last one down), and both the host and this + // kernel index elist by the drawn index directly rather than through + // glist. writing it would also need a bound the elist capacity does not + // share, since maxelectron can exceed the per-group row width + + for (int g = 0; g < ngroups; g++) d_gcount(icell,g) = 0; int nelectron = 0; for (int n = 0; n < np; n++) { const int ip = d_plist(icell,n); const int isp = d_particles[ip].ispecies; - gcount[d_species2group[isp]]++; - if (d_ionambi[ip]) nelectron++; - } - gcount[egroup] = nelectron; - - int offset = 0; - for (int g = 0; g < ngroups; g++) { - gstart[g] = offset; - gcursor[g] = offset; - if (g != egroup) offset += gcount[g]; - } + addgroup_kk(icell,d_species2group[isp],n); - int e = 0; - for (int n = 0; n < np; n++) { - const int ip = d_plist(icell,n); - const int isp = d_particles[ip].ispecies; - const int g = d_species2group[isp]; - d_glist(icell,gcursor[g]++) = n; if (d_ionambi[ip]) { - Particle::OnePart* p = &d_particles[ip]; - Particle::OnePart* ep = &d_elist(icell,e); - *ep = *p; + Particle::OnePart* ep = &d_elist(icell,nelectron); + *ep = d_particles[ip]; ep->v[0] = d_velambi(ip,0); ep->v[1] = d_velambi(ip,1); ep->v[2] = d_velambi(ip,2); ep->ispecies = ambispecies; - e++; + nelectron++; + d_gcount(icell,egroup)++; } } @@ -2087,7 +2761,8 @@ void CollideVSSKokkos::operator()(TagCollideCollisionsGroupAmbipolar< GASTALLY, continue; } const double attempt = - attempt_collision_kokkos(icell,ig,jg,gcount[ig],gcount[jg],volume,rand_gen); + attempt_collision_kokkos(icell,ig,jg,d_gcount(icell,ig), + d_gcount(icell,jg),volume,rand_gen); const int nattempt = static_cast (attempt); d_nattempt_pair(icell,ig,jg) = nattempt; if (nattempt) { @@ -2114,29 +2789,62 @@ void CollideVSSKokkos::operator()(TagCollideCollisionsGroupAmbipolar< GASTALLY, if (ig == egroup) { aig = jg; ajg = ig; } else { aig = ig; ajg = jg; } - const int ni = gcount[aig]; - const int nj = gcount[ajg]; - if (ni == 0 || nj == 0) continue; - if (aig == ajg && ni == 1) continue; + // group counts are re-read from d_gcount every time, because a + // reaction in an earlier pair may have emptied a group + + if (d_gcount(icell,aig) == 0 || d_gcount(icell,ajg) == 0) continue; + if (aig == ajg && d_gcount(icell,aig) == 1) continue; for (int iattempt = 0; iattempt < nattempt; iattempt++) { + const int ni = d_gcount(icell,aig); + const int nj = d_gcount(icell,ajg); + int i = ni * rand_gen.drand(); int j = nj * rand_gen.drand(); if (aig == ajg) while (i == j) j = nj * rand_gen.drand(); - Particle::OnePart* ipart = - &d_particles[d_plist(icell,d_glist(icell,gstart[aig]+i))]; + // ii/jj are plist indices, captured before any regrouping moves them + // for the electron side there is no plist entry: elist is indexed by + // the drawn index itself + + const int ii = d_glist(icell,aig,i); + const int jj = (ajg == egroup) ? -1 : d_glist(icell,ajg,j); + + Particle::OnePart* ipart = &d_particles[d_plist(icell,ii)]; Particle::OnePart* jpart; if (ajg == egroup) jpart = &d_elist(icell,j); - else jpart = &d_particles[d_plist(icell,d_glist(icell,gstart[ajg]+j))]; + else jpart = &d_particles[d_plist(icell,jj)]; // test if collision actually occurs if (!test_collision_kokkos(icell,aig,ajg,ipart,jpart,precoln,rand_gen)) continue; - // perform collision (non-reacting: no chemistry, no 3rd particle) - // if GASTALLY: save iorig/jorig for tally + // if recombination reaction is possible for this IJ pair + // pick a 3rd particle to participate and set cell number density + // unless boost factor turns it off, or there is no 3rd particle + // 3rd particle is never an electron since plist has no electrons + // if ajg == egroup, no need to check k for match to jj + + Particle::OnePart* recomb_part3 = NULL; + int recomb_species = -1; + double recomb_density = 0.0; + if (recombflag && d_recomb_ijflag(ipart->ispecies,jpart->ispecies)) { + if (rand_gen.drand() > recomb_boost_inverse) + recomb_species = -1; + else if (np <= 2) + recomb_species = -1; + else { + int k = np * rand_gen.drand(); + while (k == ii || k == jj) k = np * rand_gen.drand(); + recomb_part3 = &d_particles[d_plist(icell,k)]; + recomb_species = recomb_part3->ispecies; + recomb_density = np * fnum / volume; + } + } + + // perform collision + // if GASTALLY: save iorig/jorig, then trigger the tally Particle::OnePart iorig,jorig; if (GASTALLY) { @@ -2145,14 +2853,13 @@ void CollideVSSKokkos::operator()(TagCollideCollisionsGroupAmbipolar< GASTALLY, } Particle::OnePart* kpart = NULL; - Particle::OnePart* recomb_part3 = NULL; - int recomb_species = -1; - double recomb_density = 0.0; int index_kpart = 0; + const int jspecies = jpart->ispecies; setup_collision_kokkos(ipart,jpart,precoln,postcoln); - const int reactflag = perform_collision_kokkos(ipart,jpart,kpart,precoln,postcoln,rand_gen, - recomb_part3,recomb_species,recomb_density,index_kpart); + const int reactflag = + perform_collision_kokkos(ipart,jpart,kpart,precoln,postcoln,rand_gen, + recomb_part3,recomb_species,recomb_density,index_kpart); if (ATOMIC_REDUCTION == 1) Kokkos::atomic_inc(&d_ncollide_one()); @@ -2163,16 +2870,219 @@ void CollideVSSKokkos::operator()(TagCollideCollisionsGroupAmbipolar< GASTALLY, if (GASTALLY) { for (int m = 0; m < nglist_collision; m++) - glist_collision_copy[m].obj.template gas_tally_kk(icell,reactflag,&iorig,&jorig,ipart,jpart,kpart); + CVK_GLIST_COLLISION(m).template gas_tally_kk(icell,reactflag,&iorig,&jorig,ipart,jpart,kpart); for (int m = 0; m < nglist_reaction; m++) - glist_reaction_copy[m].obj.template gas_tally_kk(icell,reactflag,&iorig,&jorig,ipart,jpart,kpart); + CVK_GLIST_REACTION(m).template gas_tally_kk(icell,reactflag,&iorig,&jorig,ipart,jpart,kpart); + for (int m = 0; m < nglist_coll_tally; m++) + CVK_GLIST_COLL_TALLY(m).template gas_tally_kk(icell,reactflag,&iorig,&jorig,ipart,jpart,kpart); + for (int m = 0; m < nglist_react_tally; m++) + CVK_GLIST_REACT_TALLY(m).template gas_tally_kk(icell,reactflag,&iorig,&jorig,ipart,jpart,kpart); + } + + if (reactflag) { + if (ATOMIC_REDUCTION == 1) + Kokkos::atomic_inc(&d_nreact_one()); + else if (ATOMIC_REDUCTION == 0) + d_nreact_one()++; + else + reduce.nreact_one++; + } else continue; + + // reset ambipolar ion flags due to reaction + // must do now before the group reset below can break out of the loop + + if (ajg == egroup) + ambi_reset_kokkos(d_plist(icell,ii),-1,jspecies,index_kpart, + ipart,jpart,kpart,d_ionambi); + else + ambi_reset_kokkos(d_plist(icell,ii),d_plist(icell,jj),jspecies,index_kpart, + ipart,jpart,kpart,d_ionambi); + + // ipart may now belong to a different group + // ipart is never an electron, so aig is never egroup here + + int newgroup = d_species2group[ipart->ispecies]; + if (newgroup != aig) { + addgroup_kk(icell,newgroup,ii); + delgroup_kk(icell,aig,i); + // needed if ajg == aig and delgroup moved the J particle + if (ajg == aig && j == d_gcount(icell,aig)) j = i; + } + + // if kpart was created, add it to plist or elist and to its group + // must come before the jpart code below, since that also appends + + if (kpart) { + newgroup = d_species2group[kpart->ispecies]; + + if (newgroup != egroup) { + if (np < int(d_plist.extent(1))) { + d_plist(icell,np) = index_kpart; + addgroup_kk(icell,newgroup,np); + np++; + } else { + d_retry() = 1; + d_maxcellcount() += DELTACELLCOUNT; + rand_pool.free_state(rand_gen); + return; + } + + } else { + if (nelectron < int(d_elist.extent(1))) { + Particle::OnePart* ep = &d_elist(icell,nelectron); + *ep = *kpart; + ep->ispecies = ambispecies; + nelectron++; + d_gcount(icell,egroup)++; +#ifdef SPARTA_KOKKOS_EXACT + d_nlocal()--; +#else + const int ndelete = Kokkos::atomic_fetch_add(&d_ndelete(),1); + if (ndelete < int(d_dellist.extent(0))) { + d_dellist(ndelete) = index_kpart; + } else { + d_retry() = 1; + d_maxdelete() += DELTADELETE; + rand_pool.free_state(rand_gen); + return; + } +#endif + } else { + d_retry() = 1; + d_maxelectron() += DELTACELLCOUNT; + rand_pool.free_state(rand_gen); + return; + } + } + } + + // jpart may now be in a different group, have become an electron, + // have stopped being one, or have been destroyed. the four cases + // are Collide::collisions_group_ambipolar()'s, in its order + + if (jpart) { + newgroup = d_species2group[jpart->ispecies]; + + if (newgroup == ajg) { + // nothing to do + + } else if (ajg != egroup && newgroup != egroup) { + addgroup_kk(icell,newgroup,jj); + delgroup_kk(icell,ajg,j); + + } else if (ajg != egroup && jpart->ispecies == ambispecies) { + + // ionization: two neutrals became an ion plus an electron. + // the electron goes to elist; jpart is nulled so the block + // below removes its now-stale plist and group entries + + if (nelectron < int(d_elist.extent(1))) { + Particle::OnePart* ep = &d_elist(icell,nelectron); + *ep = *jpart; + ep->ispecies = ambispecies; + nelectron++; + d_gcount(icell,egroup)++; + jpart = NULL; + } else { + d_retry() = 1; + d_maxelectron() += DELTACELLCOUNT; + rand_pool.free_state(rand_gen); + return; + } + + } else if (ajg == egroup && jpart->ispecies != ambispecies) { + + // exchange: an ion plus an electron became two neutrals, so the + // electron becomes a real particle + + const int index = Kokkos::atomic_fetch_add(&d_nlocal(),1); + const int reallocflag = + ParticleKokkos::add_particle_kokkos(d_particles,index,0,jspecies,icell, + jpart->x,jpart->v,0.0,0.0); + if (reallocflag) { + d_retry() = 1; + d_part_grow() = 1; + rand_pool.free_state(rand_gen); + return; + } + + d_particles[index] = *jpart; + d_particles[index].id = MAXSMALLINT*rand_gen.drand(); + d_ionambi[index] = 0; + + if (nelectron-1 != j) d_elist(icell,j) = d_elist(icell,nelectron-1); + nelectron--; + d_gcount(icell,egroup)--; + + if (np < int(d_plist.extent(1))) { + d_plist(icell,np) = index; + addgroup_kk(icell,newgroup,np); + np++; + } else { + d_retry() = 1; + d_maxcellcount() += DELTACELLCOUNT; + rand_pool.free_state(rand_gen); + return; + } + } + } + + if (!jpart && jspecies == ambispecies) { + + // recombination consumed the electron: swap the last one down, + // which keeps the host's glist[egroup][k] == k invariant + + if (nelectron-1 != j) d_elist(icell,j) = d_elist(icell,nelectron-1); + nelectron--; + d_gcount(icell,egroup)--; + + } else if (!jpart) { + + // jpart was a real particle and is gone: delete it, drop it from + // its group, and swap-remove it from plist, repairing the moved + // entry's group slot through the reverse map as Collide does + + const int ndelete = Kokkos::atomic_fetch_add(&d_ndelete(),1); + if (ndelete < int(d_dellist.extent(0))) { + d_dellist(ndelete) = d_plist(icell,jj); + } else { + d_retry() = 1; + d_maxdelete() += DELTADELETE; + rand_pool.free_state(rand_gen); + return; + } + + delgroup_kk(icell,ajg,j); + + np--; + d_plist(icell,jj) = d_plist(icell,np); + if (jj < np) { + const int mg = d_p2g(icell,np,0); + const int mk = d_p2g(icell,np,1); + d_glist(icell,mg,mk) = jj; + d_p2g(icell,jj,0) = mg; + d_p2g(icell,jj,1) = mk; + } + } + + // stop attempting if either group has become too small + + const int nig = d_gcount(icell,aig); + if (nig <= 1) { + if (nig == 0) break; + if (aig == ajg) break; + } + const int njg = d_gcount(icell,ajg); + if (njg <= 1) { + if (njg == 0) break; + if (aig == ajg) break; } } } // recombine ambipolar ions with their matching electrons // by copying the (possibly scattered) electron velocity back into velambi - // electrons were created in plist order, so the Nth ion gets the Nth electron + // which ion is paired with which electron does not matter int melectron = 0; for (int n = 0; n < np; n++) { @@ -2220,12 +3130,6 @@ void CollideVSSKokkos::collisions_one_ambipolar(COLLIDE_REDUCE &reduce) grid_kk->sync(Device,CINFO_MASK); d_plist = grid_kk->d_plist; - if (react) { - ReactTCEKokkos* react_kk = (ReactTCEKokkos*) react; - if (!react_kk) - error->all(FLERR,"Must use TCE reactions with Kokkos"); - } - copymode = 1; /* ATOMIC_REDUCTION: 1 = use atomics @@ -2294,12 +3198,34 @@ void CollideVSSKokkos::collisions_one_ambipolar(COLLIDE_REDUCE &reduce) } } + // a per-event gas tally compute can force a retry of its own, and a retry + // re-runs the collision pass over the same particles. that is only sound + // if the particle list can be rolled back first, so the backup is not + // gated on react/retry when one of those computes is active + + const int tally_backup = (nglist_coll_tally || nglist_react_tally); + const int do_backup = + (react && sparta->kokkos->react_retry_flag) || tally_backup; + + if (tally_backup) rewind_gas_tally_computes(1); + while (h_retry()) { - if (react && sparta->kokkos->react_retry_flag) - backup(); + if (do_backup) backup(); + + // discard the rows an aborted attempt appended, including an attempt + // repeated for a reaction overflow rather than a tally overflow + + if (tally_backup) rewind_gas_tally_computes(0); h_retry() = 0; + // seed from the allocation, not from the Collide member: maxelectron + // starts at 0 and the kernel only overflows against d_elist.extent(1), + // so seeding it with 0 makes each retry bump the request by + // DELTACELLCOUNT from zero and re-run the whole pass until it finally + // exceeds the extent -- dozens of wasted sweeps before the realloc + + maxelectron = d_elist.extent(1); h_maxelectron() = maxelectron; h_maxdelete() = maxdelete; h_maxcellcount() = maxcellcount; @@ -2307,6 +3233,14 @@ void CollideVSSKokkos::collisions_one_ambipolar(COLLIDE_REDUCE &reduce) h_ndelete() = 0; h_nlocal() = particle->nlocal; + // h_tally_overflow is not zeroed anywhere else on this path: the reaction + // retry branch below reads maxdelete/maxcellcount/nlocal back out of + // h_scalars, so unlike UpdateKokkos it cannot bulk-zero the array. A + // pass that raised both flags would otherwise push a stale 1 back to the + // device and trigger a spurious grow plus a wasted sweep next attempt + + h_tally_overflow() = 0; + Kokkos::deep_copy(d_scalars,h_scalars); Kokkos::deep_copy(d_scalars_big,h_scalars_big); @@ -2346,11 +3280,26 @@ void CollideVSSKokkos::collisions_one_ambipolar(COLLIDE_REDUCE &reduce) Kokkos::deep_copy(h_scalars,d_scalars); Kokkos::deep_copy(h_scalars_big,d_scalars_big); + // a per-event gas tally compute ran out of room: grow it and repeat the + // pass. unlike a reaction overflow this needs no react/retry opt-in, + // and clear_gas_tally() below already discards the aborted pass + + if (h_tally_overflow() && !h_retry()) { + grow_gas_tally_computes(); + if (do_backup) restore(); + if (ngas_tally) clear_gas_tally(); + Kokkos::deep_copy(h_scalars,0); + Kokkos::deep_copy(h_scalars_big,0); + reduce = COLLIDE_REDUCE(); + h_retry() = 1; + continue; + } + if (h_retry()) { //printf("Retrying, reason %i %i %i %i !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n",h_maxelectron() > d_elist.extent(1),h_maxdelete() > d_dellist.extent(0),h_maxcellcount() > d_plist.extent(1),h_part_grow()); //printf("%i %i %i %i %i %i %i !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n",h_maxelectron(),d_elist.extent(1),h_maxdelete(),d_dellist.extent(0),h_maxcellcount(),d_plist.extent(1),h_part_grow()); - if (!sparta->kokkos->react_retry_flag) { + if (!do_backup) { error->one(FLERR,"Ran out of space in Kokkos collisions, increase react/extra" " or use react/retry"); } else @@ -2576,9 +3525,13 @@ void CollideVSSKokkos::operator()(TagCollideCollisionsOneAmbipolar< GASTALLY, AT if (GASTALLY) { for (int m = 0; m < nglist_collision; m++) - glist_collision_copy[m].obj.template gas_tally_kk(icell,reactflag,&iorig,&jorig,ipart,jpart,kpart); + CVK_GLIST_COLLISION(m).template gas_tally_kk(icell,reactflag,&iorig,&jorig,ipart,jpart,kpart); for (int m = 0; m < nglist_reaction; m++) - glist_reaction_copy[m].obj.template gas_tally_kk(icell,reactflag,&iorig,&jorig,ipart,jpart,kpart); + CVK_GLIST_REACTION(m).template gas_tally_kk(icell,reactflag,&iorig,&jorig,ipart,jpart,kpart); + for (int m = 0; m < nglist_coll_tally; m++) + CVK_GLIST_COLL_TALLY(m).template gas_tally_kk(icell,reactflag,&iorig,&jorig,ipart,jpart,kpart); + for (int m = 0; m < nglist_react_tally; m++) + CVK_GLIST_REACT_TALLY(m).template gas_tally_kk(icell,reactflag,&iorig,&jorig,ipart,jpart,kpart); } if (reactflag) { @@ -3641,6 +4594,93 @@ double CollideVSSKokkos::vibrel(int isp, double Ec) const return vibphi; } +/* ---------------------------------------------------------------------- + near neighbor search for a group pair + mirrors Collide::find_nn_group() (collide.cpp:2634). ni/nj are the two + group counts; when ig == jg the host passes the same nn array for both, + which is d_nn_igroup here +------------------------------------------------------------------------- */ + +KOKKOS_INLINE_FUNCTION +int CollideVSSKokkos::find_nn_group(rand_type &rand_gen, int icell, int i, + int ig, int jg, int ni, int nj) const +{ + int jneigh; + double dx,dy,dz,rsq; + double *xj; + + const int same = (ig == jg); + + // if same group and nj = 2, just return J = non-I particle + + if (same && nj == 2) return (i+1) % 2; + + Particle::OnePart *ipart,*jpart; + + // thresh = distance particle I moves in this timestep + + ipart = &d_particles[d_plist(icell,d_glist(icell,ig,i))]; + double *vi = ipart->v; + double *xi = ipart->x; + double threshsq = dt*dt * (vi[0]*vi[0]+vi[1]*vi[1]+vi[2]*vi[2]); + double minrsq = BIG; + + // nlimit = max # of J candidates to consider + + int nlimit = MIN(nearlimit,nj-1); + int count = 0; + + // pick a random starting J + // jneigh = collision partner when exit loop + // set to initial J as default in case no Nlimit J meets criteria + + int j = nj * rand_gen.drand(); + if (same) + while (i == j) j = nj * rand_gen.drand(); + jneigh = j; + + while (count < nlimit) { + count++; + + // skip this J if I,J last collided with each other + + const int nnj = same ? d_nn_igroup(icell,j) : d_nn_jgroup(icell,j); + if (d_nn_igroup(icell,i) == j+1 && nnj == i+1) { + j++; + if (j == nj) j = 0; + continue; + } + + // rsq = squared distance between particles I and J + // if rsq = 0.0, skip this J + // if rsq <= threshsq, this J is collision partner + // if rsq = smallest yet seen, this J is tentative collision partner + + jpart = &d_particles[d_plist(icell,d_glist(icell,jg,j))]; + xj = jpart->x; + dx = xi[0] - xj[0]; + dy = xi[1] - xj[1]; + dz = xi[2] - xj[2]; + rsq = dx*dx + dy*dy + dz*dz; + + if (rsq > 0.0) { + if (rsq <= threshsq) { + jneigh = j; + break; + } + if (rsq < minrsq) { + minrsq = rsq; + jneigh = j; + } + } + + j++; + if (j == nj) j = 0; + } + + return jneigh; +} + /* ---------------------------------------------------------------------- for particle I, find collision partner J via near neighbor algorithm always returns a J neighbor, even if not that near @@ -4050,8 +5090,23 @@ void CollideVSSKokkos::grow_percell(int n) void CollideVSSKokkos::sync(ExecutionSpace space, unsigned int mask) { if (space == Device) { - if (sparta->kokkos->auto_sync) + if (sparta->kokkos->auto_sync) { + // Automatic syncing exists because non-Kokkos code may have written the + // plain vremax/remain pointers, so the host is declared modified and + // copied down. Declaring it while the device still holds a claim is + // both a lie -- the host copy is the older one -- and fatal: Kokkos + // aborts a DualView claimed on both sides at once. collisions() claims + // the device every step and nothing calls sync(Host) in the run loop, so + // the claim stands for the whole run; the first sync(Device) made with + // auto_sync on then aborts. fix adapt is exactly that caller -- it sets + // kokkos_flag = 0, which makes ModifyKokkos turn auto_sync on around + // end_of_step(), and CollideVSSKokkos::grow_percell() syncs to the + // device from inside AdaptGrid::perform_refine(). Refresh the host + // first: a no-op when the device is clean, and the copy the host is owed + // when it is not. + sync(Host,mask); modified(Host,mask); + } if (mask & VREMAX_MASK) k_vremax.sync_device(); if (remainflag) if (mask & REMAIN_MASK) k_remain.sync_device(); @@ -4172,3 +5227,64 @@ void CollideVSSKokkos::restore() d_velambi_backup = {}; } } + +/* ---------------------------------------------------------------------- + grow every per-event gas tally compute past what the failed attempt + needed; see the same helper in UpdateKokkos +------------------------------------------------------------------------- */ + +void CollideVSSKokkos::grow_gas_tally_computes() +{ + int ncoll = 0, nreact = 0; + + for (int i = 0; i < ngas_tally; i++) { + Compute *c = update->glist_active[i]; + if (ComputeGasCollisionTallyKokkos *ckk = dynamic_cast(c)) { + ckk->grow_after_overflow(); + + // growing reallocated the compute's row buffer, so the copy the kernel + // reads still points at the old, too-small one. Without re-blitting + // it the repeated attempt overflows on the same row and the retry + // loop never terminates + +#ifdef SPARTA_KOKKOS_FIXED_LISTS + glist_coll_tally_copy[ncoll++].copy(ckk); +#else + gas_buf_blit(k_glist_coll_tally,ncoll++,ckk); +#endif + } else if (ComputeGasReactionTallyKokkos *ckk = dynamic_cast(c)) { + ckk->grow_after_overflow(); +#ifdef SPARTA_KOKKOS_FIXED_LISTS + glist_react_tally_copy[nreact++].copy(ckk); +#else + gas_buf_blit(k_glist_react_tally,nreact++,ckk); +#endif + } + } + +#ifndef SPARTA_KOKKOS_FIXED_LISTS + gas_buf_sync(k_glist_coll_tally,d_glist_coll_tally); + gas_buf_sync(k_glist_react_tally,d_glist_react_tally); +#endif +} + +/* ---------------------------------------------------------------------- + mark (mark=1) or rewind to (mark=0) the append position of every per-event + gas tally compute + an aborted attempt of the retry loop has to take back the rows it appended + before the pass re-runs; clear_gas_tally() only resets the per-grid + computes, so it does not cover these +------------------------------------------------------------------------- */ + +void CollideVSSKokkos::rewind_gas_tally_computes(int mark) +{ + for (int i = 0; i < ngas_tally; i++) { + Compute *c = update->glist_active[i]; + if (ComputeGasCollisionTallyKokkos *ckk = + dynamic_cast(c)) + { if (mark) ckk->mark_ntally(); else ckk->rewind_ntally(); } + else if (ComputeGasReactionTallyKokkos *ckk = + dynamic_cast(c)) + { if (mark) ckk->mark_ntally(); else ckk->rewind_ntally(); } + } +} diff --git a/src/KOKKOS/collide_vss_kokkos.h b/src/KOKKOS/collide_vss_kokkos.h index b55c341f1..5cd94fec3 100644 --- a/src/KOKKOS/collide_vss_kokkos.h +++ b/src/KOKKOS/collide_vss_kokkos.h @@ -34,8 +34,9 @@ CollideStyle(vss/kk,CollideVSSKokkos) #include "kokkos_copy.h" #include "compute_gas_collision_grid_kokkos.h" #include "compute_gas_reaction_grid_kokkos.h" +#include "compute_gas_collision_tally_kokkos.h" +#include "compute_gas_reaction_tally_kokkos.h" -#define KOKKOS_MAX_GLIST 4 namespace SPARTA_NS { @@ -187,12 +188,43 @@ class CollideVSSKokkos : public CollideVSS { KKCopy react_tceqk_kk_copy; int react_style; // 0=TCE, 1=QK, 2=TCEQK (set in setup) - // active gas/gas per-grid tally computes, partitioned by type + // active gas/gas tally computes, partitioned by type. Two representations, + // selected by SPARTA_KOKKOS_FIXED_LISTS (see kokkos_type.h); the kernel + // dispatch sites are written once against the CVK_* accessors below. + +#ifdef SPARTA_KOKKOS_FIXED_LISTS KKCopy glist_collision_copy[KOKKOS_MAX_GLIST]; + KKCopy glist_coll_tally_copy[KOKKOS_MAX_GLIST]; + KKCopy glist_react_tally_copy[KOKKOS_MAX_GLIST]; KKCopy glist_reaction_copy[KOKKOS_MAX_GLIST]; - int nglist_collision,nglist_reaction; ComputeGasCollisionGridKokkos tmp_compute_gas_collision_kk; ComputeGasReactionGridKokkos tmp_compute_gas_reaction_kk; + ComputeGasCollisionTallyKokkos tmp_compute_gas_coll_tally_kk; + ComputeGasReactionTallyKokkos tmp_compute_gas_react_tally_kk; + +#define CVK_GLIST_COLLISION(m) glist_collision_copy[m].obj +#define CVK_GLIST_REACTION(m) glist_reaction_copy[m].obj +#define CVK_GLIST_COLL_TALLY(m) glist_coll_tally_copy[m].obj +#define CVK_GLIST_REACT_TALLY(m) glist_react_tally_copy[m].obj + +#else + DAT::tdual_char_1d k_glist_collision, k_glist_reaction, + k_glist_coll_tally, k_glist_react_tally; + DAT::t_char_1d d_glist_collision, d_glist_reaction, + d_glist_coll_tally, d_glist_react_tally; + +#define CVK_GLIST_COLLISION(m) ((const ComputeGasCollisionGridKokkos *) d_glist_collision.data())[m] +#define CVK_GLIST_REACTION(m) ((const ComputeGasReactionGridKokkos *) d_glist_reaction.data())[m] +#define CVK_GLIST_COLL_TALLY(m) ((const ComputeGasCollisionTallyKokkos *) d_glist_coll_tally.data())[m] +#define CVK_GLIST_REACT_TALLY(m) ((const ComputeGasReactionTallyKokkos *) d_glist_react_tally.data())[m] +#endif + + int nglist_collision,nglist_reaction; + int nglist_coll_tally,nglist_react_tally; + DAT::t_int_scalar d_tally_overflow; + HAT::t_int_scalar h_tally_overflow; + void grow_gas_tally_computes(); + void rewind_gas_tally_computes(int); void setup_gas_tally(); void finish_gas_tally(); void clear_gas_tally(); @@ -203,7 +235,67 @@ class CollideVSSKokkos : public CollideVSS { // group collision scratch (ngroups > 1) DAT::t_int_1d d_species2group; - DAT::t_int_2d d_glist; + // reacting group collisions mutate group membership inside the kernel, so + // the per-group lists cannot be one group-contiguous row per cell as they + // were when only the non-reacting case was supported. Each group gets its + // own region of capacity d_plist.extent(1) -- a group can never hold more + // than the cell does -- and d_p2g is the reverse map the host keeps in + // Collide::p2g, needed so a swap-remove can fix the moved entry's owner. + + Kokkos::View d_glist; // (cell, group, k) -> plist index + Kokkos::View d_p2g; // (cell, plist index) -> group, k + + // per-group counters, formerly per-thread stack arrays dimensioned by a + // compile-time MAXGROUP. Both group kernels are one work item per grid + // cell (RangePolicy over 0..nglocal indexed by icell), so icell is already + // a unique, deterministic, contention-free row index -- no UniqueToken + // needed. Sized (nglocal, ngroups) alongside d_glist, which they cost + // 1/d_plist.extent(1) as much as. + // both group kernels build their lists with addgroup_kk(), which keeps + // d_gcount and d_p2g in step, so no separate fill cursor is needed. + + Kokkos::View d_gcount; // (cell, group) -> # in group + + // near-neighbor partner history for the two groups of the current pair; + // the host reallocates these per pair via set_nn_group() + + DAT::t_int_2d d_nn_igroup; + DAT::t_int_2d d_nn_jgroup; + + public: + + // mirror Collide::addgroup / delgroup (collide.h:157-179) exactly, including + // the swap-with-last order: a reaction that rebins a particle changes which + // index a later random draw lands on, so any deviation diverges from the + // host rather than merely reordering + + // the group counts live in d_gcount(icell,*), so icell is all these need + + KOKKOS_INLINE_FUNCTION + void addgroup_kk(const int icell, const int igroup, const int pindex) const + { + const int ng = d_gcount(icell,igroup); + d_glist(icell,igroup,ng) = pindex; + d_p2g(icell,pindex,0) = igroup; + d_p2g(icell,pindex,1) = ng; + d_gcount(icell,igroup)++; + } + + KOKKOS_INLINE_FUNCTION + void delgroup_kk(const int icell, const int igroup, const int i) const + { + const int ng = d_gcount(icell,igroup); + if (i < ng-1) { + d_glist(icell,igroup,i) = d_glist(icell,igroup,ng-1); + const int pindex = d_glist(icell,igroup,i); + d_p2g(icell,pindex,0) = igroup; + d_p2g(icell,pindex,1) = i; + } + d_gcount(icell,igroup)--; + } + + private: + Kokkos::View d_nattempt_pair; DAT::t_int_1d d_ewhich; @@ -226,7 +318,7 @@ class CollideVSSKokkos : public CollideVSS { // bigint scalars = per-step statistics counters, can exceed 2^31 // in one step at large per-proc particle counts - typedef Kokkos::DualView tdual_int_8; + typedef Kokkos::DualView tdual_int_8; typedef tdual_int_8::t_dev t_int_8; typedef tdual_int_8::t_host t_host_int_8; t_int_8 d_scalars; @@ -358,6 +450,11 @@ class CollideVSSKokkos : public CollideVSS { KOKKOS_INLINE_FUNCTION int find_nn(rand_type &, int, int, int) const; + void grow_group_lists(); + + KOKKOS_INLINE_FUNCTION + int find_nn_group(rand_type &, int, int, int, int, int, int) const; + void backup(); void restore(); diff --git a/src/KOKKOS/comm_kokkos.cpp b/src/KOKKOS/comm_kokkos.cpp index f40d469c0..469ca5c1d 100644 --- a/src/KOKKOS/comm_kokkos.cpp +++ b/src/KOKKOS/comm_kokkos.cpp @@ -198,9 +198,21 @@ int CommKokkos::migrate_particles(int nmigrate, int *plist, const DAT::t_int_1d d_particles = particle_kk->k_particles.view_device(); if (gpu_aware_flag && !ncustom) { - iparticle_kk-> - exchange_uniform(d_sbuf,nbytes_total, - (char *) (d_particles.data()+particle->nlocal),d_rbuf); + + // received particles land straight in the particle list, past nlocal. + // exchange_uniform() also copies the self datums itself, into the view it + // is handed, so that view has to describe the same buffer the receives + // target. d_rbuf is only allocated in the branch below, so passing it + // here handed the self copy a default constructed view -- extent 0, null + // data -- to index. It goes unnoticed because migrate_particles never + // produces a self datum: a particle is on the migrate list precisely + // because its new cell belongs to another proc, so the destination list + // never contains me. Describe the real destination instead, the way every + // other caller of exchange_uniform already does. + + char *d_pdest = (char *) (d_particles.data()+particle->nlocal); + DAT::t_char_1d d_pbuf(d_pdest,(size_t)nrecv*nbytes_total); + iparticle_kk->exchange_uniform(d_sbuf,nbytes_total,d_pdest,d_pbuf); } else { // allocate exact buffer size to reduce GPU <--> CPU memory transfer diff --git a/src/KOKKOS/compute_dt_grid_kokkos.cpp b/src/KOKKOS/compute_dt_grid_kokkos.cpp index fe2d08758..182916b8c 100644 --- a/src/KOKKOS/compute_dt_grid_kokkos.cpp +++ b/src/KOKKOS/compute_dt_grid_kokkos.cpp @@ -108,6 +108,9 @@ void ComputeDtGridKokkos::compute_per_grid_kokkos() if (!ftau->kokkos_flag) error->all(FLERR,"Cannot (yet) use non-Kokkos fixes with compute dt/grid/kk"); KokkosBase* computeKKBase = dynamic_cast(ftau); + // a fix keeps its per-grid output between invocations and grid migration + // can leave it current on the host alone, so ask for the device copy + if (computeKKBase) computeKKBase->sync_pergrid_device_kokkos(); if (tau_index == 0) d_tau_vector = computeKKBase->d_vector_grid; else { @@ -141,6 +144,9 @@ void ComputeDtGridKokkos::compute_per_grid_kokkos() if (!ftemp->kokkos_flag) error->all(FLERR,"Cannot (yet) use non-Kokkos fixes with compute dt/grid/kk"); KokkosBase* computeKKBase = dynamic_cast(ftemp); + // a fix keeps its per-grid output between invocations and grid migration + // can leave it current on the host alone, so ask for the device copy + if (computeKKBase) computeKKBase->sync_pergrid_device_kokkos(); if (temp_index == 0) d_temp_vector = computeKKBase->d_vector_grid; else { @@ -174,6 +180,9 @@ void ComputeDtGridKokkos::compute_per_grid_kokkos() if (!fusq->kokkos_flag) error->all(FLERR,"Cannot (yet) use non-Kokkos fixes with compute dt/grid/kk"); KokkosBase* computeKKBase = dynamic_cast(fusq); + // a fix keeps its per-grid output between invocations and grid migration + // can leave it current on the host alone, so ask for the device copy + if (computeKKBase) computeKKBase->sync_pergrid_device_kokkos(); if (usq_index == 0) d_usq_vector = computeKKBase->d_vector_grid; else { @@ -207,6 +216,9 @@ void ComputeDtGridKokkos::compute_per_grid_kokkos() if (!fvsq->kokkos_flag) error->all(FLERR,"Cannot (yet) use non-Kokkos fixes with compute dt/grid/kk"); KokkosBase* computeKKBase = dynamic_cast(fvsq); + // a fix keeps its per-grid output between invocations and grid migration + // can leave it current on the host alone, so ask for the device copy + if (computeKKBase) computeKKBase->sync_pergrid_device_kokkos(); if (vsq_index == 0) d_vsq_vector = computeKKBase->d_vector_grid; else { @@ -240,6 +252,9 @@ void ComputeDtGridKokkos::compute_per_grid_kokkos() if (!fwsq->kokkos_flag) error->all(FLERR,"Cannot (yet) use non-Kokkos fixes with compute dt/grid/kk"); KokkosBase* computeKKBase = dynamic_cast(fwsq); + // a fix keeps its per-grid output between invocations and grid migration + // can leave it current on the host alone, so ask for the device copy + if (computeKKBase) computeKKBase->sync_pergrid_device_kokkos(); if (wsq_index == 0) d_wsq_vector = computeKKBase->d_vector_grid; else { diff --git a/src/KOKKOS/compute_fft_grid_kokkos.cpp b/src/KOKKOS/compute_fft_grid_kokkos.cpp index cc33816e5..c2620b76e 100644 --- a/src/KOKKOS/compute_fft_grid_kokkos.cpp +++ b/src/KOKKOS/compute_fft_grid_kokkos.cpp @@ -207,6 +207,9 @@ void ComputeFFTGridKokkos::compute_per_grid_kokkos() Fix *fix = modify->fix[vidx]; KokkosBase* fixKKBase = dynamic_cast(fix); + // a fix keeps its per-grid output between invocations and grid migration + // can leave it current on the host alone, so ask for the device copy + if (fixKKBase) fixKKBase->sync_pergrid_device_kokkos(); if (!fixKKBase || !fix->kokkos_flag || !fix->per_grid_flag) error->all(FLERR,"Unsupported fix used by compute fft/grid/kk"); @@ -526,7 +529,13 @@ void ComputeFFTGridKokkos::fft_create() // create FFT plan - int collective_flag = 0; // not yet supported in Kokkos version + // collective remap is not implemented in the Kokkos remap (see the comment + // in RemapKokkos2d::remap_2d_kokkos). this matches the host on every + // platform that is not Blue Gene: ComputeFFTGrid sets collective_flag + // from #ifdef __bg__ and uses 0 otherwise, so there is no user-visible + // setting being dropped here + + int collective_flag = 0; int gpu_aware_flag = sparta->kokkos->gpu_aware_flag; int tmp; @@ -710,7 +719,7 @@ void ComputeFFTGridKokkos::print_FFT_info() { if (comm->me == 0) { char str[64]; - sprintf(str,"Using " SPARTA_FFT_PREC " precision " SPARTA_FFT_KOKKOS_LIB " for FFTs\n"); + snprintf(str,sizeof(str),"Using " SPARTA_FFT_PREC " precision " SPARTA_FFT_KOKKOS_LIB " for FFTs\n"); if (screen) fprintf(screen,"%s",str); if (logfile) fprintf(logfile,"%s",str); } diff --git a/src/KOKKOS/compute_gas_collision_tally_kokkos.cpp b/src/KOKKOS/compute_gas_collision_tally_kokkos.cpp new file mode 100644 index 000000000..ec4f7f345 --- /dev/null +++ b/src/KOKKOS/compute_gas_collision_tally_kokkos.cpp @@ -0,0 +1,179 @@ +/* ---------------------------------------------------------------------- + SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer + http://sparta.github.io + Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov + Sandia National Laboratories + + Copyright (2014) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level SPARTA directory. +------------------------------------------------------------------------- */ + +#include "compute_gas_collision_tally_kokkos.h" +#include +#include "particle_kokkos.h" +#include "grid_kokkos.h" +#include "update.h" +#include "domain.h" +#include "mixture.h" +#include "memory_kokkos.h" +#include "sparta_masks.h" +#include "error.h" + +using namespace SPARTA_NS; + +#define DELTA 4096 + +/* ---------------------------------------------------------------------- */ + +ComputeGasCollisionTallyKokkos::ComputeGasCollisionTallyKokkos(SPARTA *sparta, + int narg, char **arg) : + ComputeGasCollisionTally(sparta, narg, arg) +{ + kokkos_flag = 1; + ntally_mark = 0; + + d_ntally = DAT::t_int_scalar("gas/collision/tally/kk:ntally"); + h_ntally = HAT::t_int_scalar("gas/collision/tally/kk:ntally_mirror"); + + // flatten the value list once; it never changes after construction + + DAT::tdual_int_1d k_which("gas/collision/tally/kk:which",nvalue); + for (int m = 0; m < nvalue; m++) k_which.view_host()[m] = which[m]; + k_which.modify_host(); + k_which.sync_device(); + d_which = k_which.view_device(); + + maxtally = DELTA; + maxtally_host = 0; + MemKK::realloc_kokkos(k_array_tally,"gas/collision/tally/kk:array_tally", + maxtally,nvalue); + d_array_tally = k_array_tally.view_device(); +} + +/* ---------------------------------------------------------------------- */ + +ComputeGasCollisionTallyKokkos::ComputeGasCollisionTallyKokkos(SPARTA *sparta) : + ComputeGasCollisionTally(sparta) +{ + copy = 1; + ntally_mark = 0; +} + +/* ---------------------------------------------------------------------- */ + +ComputeGasCollisionTallyKokkos::~ComputeGasCollisionTallyKokkos() +{ + if (copy || copymode) return; + + // the host base class frees array_tally, which this class never allocated + + memory->destroy(array_tally); + array_tally = NULL; + maxtally = 0; +} + +/* ---------------------------------------------------------------------- */ + +void ComputeGasCollisionTallyKokkos::clear() +{ + ntally = 0; +} + +/* ---------------------------------------------------------------------- + called by CollideVSSKokkos before the collision kernel +------------------------------------------------------------------------- */ + +void ComputeGasCollisionTallyKokkos::pre_gas_tally() +{ + GridKokkos* grid_kk = (GridKokkos*) grid; + grid_kk->sync(Device,CELL_MASK|CINFO_MASK); + d_cells = grid_kk->k_cells.view_device(); + d_cinfo = grid_kk->k_cinfo.view_device(); + + ParticleKokkos* particle_kk = (ParticleKokkos*) particle; + particle_kk->sync(Device,SPECIES_MASK); + d_s2g = particle_kk->k_species2group.view_device(); + + Kokkos::deep_copy(d_ntally,0); +} + +/* ---------------------------------------------------------------------- + called by CollideVSSKokkos after the collision kernel + bring the row count and the rows themselves to the host, where + dump tally and Compute::tallyinfo() read them +------------------------------------------------------------------------- */ + +void ComputeGasCollisionTallyKokkos::post_gas_tally() +{ + Kokkos::deep_copy(h_ntally,d_ntally); + ntally = h_ntally(); + + // an overflowed attempt is discarded and repeated by CollideVSSKokkos, so do + // not publish its partial rows. the device counter kept climbing past + // the end of the buffer, so ntally is not a row count here -- leaving it + // in the host base class would make dump tally read that many rows out + // of an array that never held them + + if (ntally > (int) d_array_tally.extent(0)) { ntally = 0; return; } + + // the host base class hands out array_tally, row-indexed to ntally + // (compute_surf_collision_tally.cpp:172). Publish only the rows this + // step produced: k_array_tally is sized to maxtally, a high-water mark, + // so sync_host() moved the whole buffer and the scalar loop then touched + // every element of it again. The host half of the dual view has no + // other reader, so copy the device rows straight into array_tally -- + // d_array_tally is LayoutRight and Memory::create hands back one + // contiguous row-major block with the same nvalue row stride, so an + // unmanaged wrap of the leading ntally rows lines up exactly. Keep the + // allocation on its own high-water mark rather than free/malloc-ing it + // every step. + + if (ntally) { + static_assert(std::is_same::value, + "array_tally is double**, so F_FLOAT must be double " + "for the unmanaged wrap below to alias it"); + if (ntally > maxtally_host) { + memory->destroy(array_tally); + maxtally_host = MAX(ntally,maxtally); + memory->create(array_tally,maxtally_host,nvalue, + "gas/collision/tally/kk:array_tally_host"); + } + Kokkos::View > + h_rows(array_tally[0],ntally,nvalue); + Kokkos::deep_copy(h_rows, + Kokkos::subview(d_array_tally, + Kokkos::make_pair(0,ntally), + Kokkos::ALL())); + } +} + +/* ---------------------------------------------------------------------- + grow the device row buffer to hold at least N rows + called by CollideVSSKokkos when an attempt overflowed, before repeating it +------------------------------------------------------------------------- */ + +void ComputeGasCollisionTallyKokkos::grow_tally_kokkos(int n) +{ + if (n <= (int) d_array_tally.extent(0)) return; + + // grow past what the failed attempt asked for, so a step whose collision + // count is still climbing does not repeat the move again and again + + maxtally = MAX(n + DELTA, (int)(1.5*n)); + + // resize, not realloc: MemKK::realloc_kokkos drops the old allocation and + // allocates WithoutInitializing, so every existing row becomes garbage. + // The rows below ntally_mark were written by earlier migration iterations + // of this same step and must survive -- rewind_ntally() only takes the + // count back to the mark, and the re-run appends from there, so a + // discarded prefix is published as uninitialized tally output. + // DualView::resize preserves contents, as memoryKK->grow_kokkos does. + + k_array_tally.resize(maxtally,nvalue); + d_array_tally = k_array_tally.view_device(); +} diff --git a/src/KOKKOS/compute_gas_collision_tally_kokkos.h b/src/KOKKOS/compute_gas_collision_tally_kokkos.h new file mode 100644 index 000000000..c2c93ea9d --- /dev/null +++ b/src/KOKKOS/compute_gas_collision_tally_kokkos.h @@ -0,0 +1,144 @@ +/* ---------------------------------------------------------------------- + SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer + http://sparta.github.io + Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov + Sandia National Laboratories + + Copyright (2014) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level SPARTA directory. +------------------------------------------------------------------------- */ + +#ifdef COMPUTE_CLASS + +ComputeStyle(gas/collision/tally/kk,ComputeGasCollisionTallyKokkos) + +#else + +#ifndef SPARTA_COMPUTE_GAS_COLLISION_TALLY_KOKKOS_H +#define SPARTA_COMPUTE_GAS_COLLISION_TALLY_KOKKOS_H + +#include "compute_gas_collision_tally.h" +#include "kokkos_base.h" +#include "kokkos_type.h" + +namespace SPARTA_NS { + +// unlike the per-grid and per-surf tally computes, this one appends one row +// per collision event, so the row count is not known before the collide +// kernel runs. rows are claimed with an atomic counter; if a claim lands +// past the end of the buffer the row is dropped and d_tally_overflow is +// raised, which makes CollideVSSKokkos grow every such compute and repeat the +// collision pass. truncating instead would silently corrupt dump tally output. + +class ComputeGasCollisionTallyKokkos : public ComputeGasCollisionTally, public KokkosBase { + public: + ComputeGasCollisionTallyKokkos(class SPARTA *, int, char **); + ComputeGasCollisionTallyKokkos(class SPARTA *); + ~ComputeGasCollisionTallyKokkos() override; + + void clear() override; + + // called by CollideVSSKokkos around the collision kernel + + void pre_gas_tally(); + void post_gas_tally(); + void grow_tally_kokkos(int); + + // grow to what the overflowed attempt actually needed; the device counter + // kept counting past the end of the buffer, so it is that number + + void grow_after_overflow() + { + Kokkos::deep_copy(h_ntally,d_ntally); + grow_tally_kokkos(h_ntally()); + } + + // every attempt of the retry loop re-runs the whole pass, so the rows the + // aborted attempt appended have to be taken back before the next one. + // Not by zeroing: the move kernel runs once per migration iteration and + // the tally accumulates across all of them, so an attempt rewinds to + // where its own pass started, which mark_ntally() records. + + void mark_ntally() + { + Kokkos::deep_copy(h_ntally,d_ntally); + ntally_mark = h_ntally(); + } + + void rewind_ntally() { Kokkos::deep_copy(d_ntally,ntally_mark); } + + DAT::t_int_scalar d_overflow; // set by CollideVSSKokkos each step + + template + KOKKOS_INLINE_FUNCTION + void gas_tally_kk(int icell, int reaction, + Particle::OnePart *iorig, Particle::OnePart *jorig, + Particle::OnePart *ip, Particle::OnePart *jp, + Particle::OnePart * /*kp*/) const + { + // this compute tallies only collisions that induce no reaction; + // reactions belong to compute gas/reaction/tally + + if (reaction) return; + + if (!(d_cinfo[icell].mask & groupbit)) return; + if (d_s2g(imix,iorig->ispecies) < 0) return; + if (d_s2g(imix,jorig->ispecies) < 0) return; + + const int itally = Kokkos::atomic_fetch_add(&d_ntally(),1); + if (itally >= (int) d_array_tally.extent(0)) { + d_overflow() = 1; + return; + } + + for (int m = 0; m < nvalue; m++) { + switch (d_which[m]) { + case IDCELL: d_array_tally(itally,m) = ubuf(d_cells[icell].id).d; break; + case ID1: d_array_tally(itally,m) = ubuf(ip->id).d; break; + case ID2: d_array_tally(itally,m) = ubuf(jp->id).d; break; + case TYPE1: d_array_tally(itally,m) = ubuf(ip->ispecies+1).d; break; + case TYPE2: d_array_tally(itally,m) = ubuf(jp->ispecies+1).d; break; + case VX1PRE: d_array_tally(itally,m) = iorig->v[0]; break; + case VY1PRE: d_array_tally(itally,m) = iorig->v[1]; break; + case VZ1PRE: d_array_tally(itally,m) = iorig->v[2]; break; + case VX2PRE: d_array_tally(itally,m) = jorig->v[0]; break; + case VY2PRE: d_array_tally(itally,m) = jorig->v[1]; break; + case VZ2PRE: d_array_tally(itally,m) = jorig->v[2]; break; + case VX1POST: d_array_tally(itally,m) = ip->v[0]; break; + case VY1POST: d_array_tally(itally,m) = ip->v[1]; break; + case VZ1POST: d_array_tally(itally,m) = ip->v[2]; break; + case VX2POST: d_array_tally(itally,m) = jp->v[0]; break; + case VY2POST: d_array_tally(itally,m) = jp->v[1]; break; + case VZ2POST: d_array_tally(itally,m) = jp->v[2]; break; + } + } + } + + private: + // must match the enum in compute_gas_collision_tally.cpp + + enum{IDCELL,ID1,ID2,TYPE1,TYPE2,VX1PRE,VY1PRE,VZ1PRE,VX2PRE,VY2PRE,VZ2PRE, + VX1POST,VY1POST,VZ1POST,VX2POST,VY2POST,VZ2POST}; + + int maxtally_host; // rows array_tally is allocated for + DAT::tdual_float_2d_lr k_array_tally; + DAT::t_float_2d_lr d_array_tally; + DAT::t_int_scalar d_ntally; + int ntally_mark; + HAT::t_int_scalar h_ntally; + + DAT::t_int_1d d_which; + DAT::t_int_2d d_s2g; + + t_cell_1d d_cells; + t_cinfo_1d d_cinfo; +}; + +} + +#endif +#endif diff --git a/src/KOKKOS/compute_gas_reaction_tally_kokkos.cpp b/src/KOKKOS/compute_gas_reaction_tally_kokkos.cpp new file mode 100644 index 000000000..d21d20c5a --- /dev/null +++ b/src/KOKKOS/compute_gas_reaction_tally_kokkos.cpp @@ -0,0 +1,179 @@ +/* ---------------------------------------------------------------------- + SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer + http://sparta.github.io + Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov + Sandia National Laboratories + + Copyright (2014) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level SPARTA directory. +------------------------------------------------------------------------- */ + +#include "compute_gas_reaction_tally_kokkos.h" +#include +#include "particle_kokkos.h" +#include "grid_kokkos.h" +#include "update.h" +#include "domain.h" +#include "mixture.h" +#include "memory_kokkos.h" +#include "sparta_masks.h" +#include "error.h" + +using namespace SPARTA_NS; + +#define DELTA 4096 + +/* ---------------------------------------------------------------------- */ + +ComputeGasReactionTallyKokkos::ComputeGasReactionTallyKokkos(SPARTA *sparta, + int narg, char **arg) : + ComputeGasReactionTally(sparta, narg, arg) +{ + kokkos_flag = 1; + ntally_mark = 0; + + d_ntally = DAT::t_int_scalar("gas/reaction/tally/kk:ntally"); + h_ntally = HAT::t_int_scalar("gas/reaction/tally/kk:ntally_mirror"); + + // flatten the value list once; it never changes after construction + + DAT::tdual_int_1d k_which("gas/reaction/tally/kk:which",nvalue); + for (int m = 0; m < nvalue; m++) k_which.view_host()[m] = which[m]; + k_which.modify_host(); + k_which.sync_device(); + d_which = k_which.view_device(); + + maxtally = DELTA; + maxtally_host = 0; + MemKK::realloc_kokkos(k_array_tally,"gas/reaction/tally/kk:array_tally", + maxtally,nvalue); + d_array_tally = k_array_tally.view_device(); +} + +/* ---------------------------------------------------------------------- */ + +ComputeGasReactionTallyKokkos::ComputeGasReactionTallyKokkos(SPARTA *sparta) : + ComputeGasReactionTally(sparta) +{ + copy = 1; + ntally_mark = 0; +} + +/* ---------------------------------------------------------------------- */ + +ComputeGasReactionTallyKokkos::~ComputeGasReactionTallyKokkos() +{ + if (copy || copymode) return; + + // the host base class frees array_tally, which this class never allocated + + memory->destroy(array_tally); + array_tally = NULL; + maxtally = 0; +} + +/* ---------------------------------------------------------------------- */ + +void ComputeGasReactionTallyKokkos::clear() +{ + ntally = 0; +} + +/* ---------------------------------------------------------------------- + called by CollideVSSKokkos before the collision kernel +------------------------------------------------------------------------- */ + +void ComputeGasReactionTallyKokkos::pre_gas_tally() +{ + GridKokkos* grid_kk = (GridKokkos*) grid; + grid_kk->sync(Device,CELL_MASK|CINFO_MASK); + d_cells = grid_kk->k_cells.view_device(); + d_cinfo = grid_kk->k_cinfo.view_device(); + + ParticleKokkos* particle_kk = (ParticleKokkos*) particle; + particle_kk->sync(Device,SPECIES_MASK); + d_s2g = particle_kk->k_species2group.view_device(); + + Kokkos::deep_copy(d_ntally,0); +} + +/* ---------------------------------------------------------------------- + called by CollideVSSKokkos after the collision kernel + bring the row count and the rows themselves to the host, where + dump tally and Compute::tallyinfo() read them +------------------------------------------------------------------------- */ + +void ComputeGasReactionTallyKokkos::post_gas_tally() +{ + Kokkos::deep_copy(h_ntally,d_ntally); + ntally = h_ntally(); + + // an overflowed attempt is discarded and repeated by CollideVSSKokkos, so do + // not publish its partial rows. the device counter kept climbing past + // the end of the buffer, so ntally is not a row count here -- leaving it + // in the host base class would make dump tally read that many rows out + // of an array that never held them + + if (ntally > (int) d_array_tally.extent(0)) { ntally = 0; return; } + + // the host base class hands out array_tally, row-indexed to ntally + // (compute_surf_collision_tally.cpp:172). Publish only the rows this + // step produced: k_array_tally is sized to maxtally, a high-water mark, + // so sync_host() moved the whole buffer and the scalar loop then touched + // every element of it again. The host half of the dual view has no + // other reader, so copy the device rows straight into array_tally -- + // d_array_tally is LayoutRight and Memory::create hands back one + // contiguous row-major block with the same nvalue row stride, so an + // unmanaged wrap of the leading ntally rows lines up exactly. Keep the + // allocation on its own high-water mark rather than free/malloc-ing it + // every step. + + if (ntally) { + static_assert(std::is_same::value, + "array_tally is double**, so F_FLOAT must be double " + "for the unmanaged wrap below to alias it"); + if (ntally > maxtally_host) { + memory->destroy(array_tally); + maxtally_host = MAX(ntally,maxtally); + memory->create(array_tally,maxtally_host,nvalue, + "gas/reaction/tally/kk:array_tally_host"); + } + Kokkos::View > + h_rows(array_tally[0],ntally,nvalue); + Kokkos::deep_copy(h_rows, + Kokkos::subview(d_array_tally, + Kokkos::make_pair(0,ntally), + Kokkos::ALL())); + } +} + +/* ---------------------------------------------------------------------- + grow the device row buffer to hold at least N rows + called by CollideVSSKokkos when an attempt overflowed, before repeating it +------------------------------------------------------------------------- */ + +void ComputeGasReactionTallyKokkos::grow_tally_kokkos(int n) +{ + if (n <= (int) d_array_tally.extent(0)) return; + + // grow past what the failed attempt asked for, so a step whose collision + // count is still climbing does not repeat the move again and again + + maxtally = MAX(n + DELTA, (int)(1.5*n)); + + // resize, not realloc: MemKK::realloc_kokkos drops the old allocation and + // allocates WithoutInitializing, so every existing row becomes garbage. + // The rows below ntally_mark were written by earlier migration iterations + // of this same step and must survive -- rewind_ntally() only takes the + // count back to the mark, and the re-run appends from there, so a + // discarded prefix is published as uninitialized tally output. + // DualView::resize preserves contents, as memoryKK->grow_kokkos does. + + k_array_tally.resize(maxtally,nvalue); + d_array_tally = k_array_tally.view_device(); +} diff --git a/src/KOKKOS/compute_gas_reaction_tally_kokkos.h b/src/KOKKOS/compute_gas_reaction_tally_kokkos.h new file mode 100644 index 000000000..77445e30f --- /dev/null +++ b/src/KOKKOS/compute_gas_reaction_tally_kokkos.h @@ -0,0 +1,163 @@ +/* ---------------------------------------------------------------------- + SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer + http://sparta.github.io + Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov + Sandia National Laboratories + + Copyright (2014) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level SPARTA directory. +------------------------------------------------------------------------- */ + +#ifdef COMPUTE_CLASS + +ComputeStyle(gas/reaction/tally/kk,ComputeGasReactionTallyKokkos) + +#else + +#ifndef SPARTA_COMPUTE_GAS_REACTION_TALLY_KOKKOS_H +#define SPARTA_COMPUTE_GAS_REACTION_TALLY_KOKKOS_H + +#include "compute_gas_reaction_tally.h" +#include "kokkos_base.h" +#include "kokkos_type.h" + +namespace SPARTA_NS { + +// unlike the per-grid and per-surf tally computes, this one appends one row +// per collision event, so the row count is not known before the collide +// kernel runs. rows are claimed with an atomic counter; if a claim lands +// past the end of the buffer the row is dropped and d_tally_overflow is +// raised, which makes CollideVSSKokkos grow every such compute and repeat the +// collision pass. truncating instead would silently corrupt dump tally output. + +class ComputeGasReactionTallyKokkos : public ComputeGasReactionTally, public KokkosBase { + public: + ComputeGasReactionTallyKokkos(class SPARTA *, int, char **); + ComputeGasReactionTallyKokkos(class SPARTA *); + ~ComputeGasReactionTallyKokkos() override; + + void clear() override; + + // called by CollideVSSKokkos around the collision kernel + + void pre_gas_tally(); + void post_gas_tally(); + void grow_tally_kokkos(int); + + // grow to what the overflowed attempt actually needed; the device counter + // kept counting past the end of the buffer, so it is that number + + void grow_after_overflow() + { + Kokkos::deep_copy(h_ntally,d_ntally); + grow_tally_kokkos(h_ntally()); + } + + // every attempt of the retry loop re-runs the whole pass, so the rows the + // aborted attempt appended have to be taken back before the next one. + // Not by zeroing: the move kernel runs once per migration iteration and + // the tally accumulates across all of them, so an attempt rewinds to + // where its own pass started, which mark_ntally() records. + + void mark_ntally() + { + Kokkos::deep_copy(h_ntally,d_ntally); + ntally_mark = h_ntally(); + } + + void rewind_ntally() { Kokkos::deep_copy(d_ntally,ntally_mark); } + + DAT::t_int_scalar d_overflow; // set by CollideVSSKokkos each step + + template + KOKKOS_INLINE_FUNCTION + void gas_tally_kk(int icell, int reaction, + Particle::OnePart *iorig, Particle::OnePart *jorig, + Particle::OnePart *ip, Particle::OnePart *jp, + Particle::OnePart *kp) const + { + // this compute tallies only collisions that induce a reaction; + // plain collisions belong to compute gas/collision/tally + + if (!reaction) return; + + if (!(d_cinfo[icell].mask & groupbit)) return; + if (d_s2g(imix,iorig->ispecies) < 0) return; + if (d_s2g(imix,jorig->ispecies) < 0) return; + + const int itally = Kokkos::atomic_fetch_add(&d_ntally(),1); + if (itally >= (int) d_array_tally.extent(0)) { + d_overflow() = 1; + return; + } + + for (int m = 0; m < nvalue; m++) { + switch (d_which[m]) { + case REACTION: d_array_tally(itally,m) = ubuf(reaction).d; break; + + case IDCELL: d_array_tally(itally,m) = ubuf(d_cells[icell].id).d; break; + case ID1PRE: d_array_tally(itally,m) = ubuf(iorig->id).d; break; + case ID2PRE: d_array_tally(itally,m) = ubuf(jorig->id).d; break; + case ID1POST: d_array_tally(itally,m) = ubuf(ip->id).d; break; + case ID2POST: + d_array_tally(itally,m) = (jp == NULL) ? ubuf(0).d : ubuf(jp->id).d; break; + case ID3POST: + d_array_tally(itally,m) = (kp == NULL) ? ubuf(0).d : ubuf(kp->id).d; break; + + case TYPE1PRE: d_array_tally(itally,m) = ubuf(iorig->ispecies+1).d; break; + case TYPE2PRE: d_array_tally(itally,m) = ubuf(jorig->ispecies+1).d; break; + case TYPE1POST: d_array_tally(itally,m) = ubuf(ip->ispecies+1).d; break; + case TYPE2POST: + d_array_tally(itally,m) = (jp == NULL) ? ubuf(0).d : ubuf(jp->ispecies+1).d; break; + case TYPE3POST: + d_array_tally(itally,m) = (kp == NULL) ? ubuf(0).d : ubuf(kp->ispecies+1).d; break; + + case VX1PRE: d_array_tally(itally,m) = iorig->v[0]; break; + case VY1PRE: d_array_tally(itally,m) = iorig->v[1]; break; + case VZ1PRE: d_array_tally(itally,m) = iorig->v[2]; break; + case VX2PRE: d_array_tally(itally,m) = jorig->v[0]; break; + case VY2PRE: d_array_tally(itally,m) = jorig->v[1]; break; + case VZ2PRE: d_array_tally(itally,m) = jorig->v[2]; break; + + case VX1POST: d_array_tally(itally,m) = ip->v[0]; break; + case VY1POST: d_array_tally(itally,m) = ip->v[1]; break; + case VZ1POST: d_array_tally(itally,m) = ip->v[2]; break; + case VX2POST: d_array_tally(itally,m) = (jp == NULL) ? 0.0 : jp->v[0]; break; + case VY2POST: d_array_tally(itally,m) = (jp == NULL) ? 0.0 : jp->v[1]; break; + case VZ2POST: d_array_tally(itally,m) = (jp == NULL) ? 0.0 : jp->v[2]; break; + case VX3POST: d_array_tally(itally,m) = (kp == NULL) ? 0.0 : kp->v[0]; break; + case VY3POST: d_array_tally(itally,m) = (kp == NULL) ? 0.0 : kp->v[1]; break; + case VZ3POST: d_array_tally(itally,m) = (kp == NULL) ? 0.0 : kp->v[2]; break; + } + } + } + + private: + // must match the enum in compute_gas_reaction_tally.cpp + + enum{REACTION,IDCELL,ID1PRE,ID2PRE,ID1POST,ID2POST,ID3POST,TYPE1PRE,TYPE2PRE, + TYPE1POST,TYPE2POST,TYPE3POST,VX1PRE,VY1PRE,VZ1PRE,VX2PRE,VY2PRE,VZ2PRE, + VX1POST,VY1POST,VZ1POST,VX2POST,VY2POST,VZ2POST,VX3POST,VY3POST,VZ3POST}; + + int maxtally_host; // rows array_tally is allocated for + DAT::tdual_float_2d_lr k_array_tally; + DAT::t_float_2d_lr d_array_tally; + DAT::t_int_scalar d_ntally; + int ntally_mark; + HAT::t_int_scalar h_ntally; + + DAT::t_int_1d d_which; + DAT::t_int_2d d_s2g; + + t_cell_1d d_cells; + t_cinfo_1d d_cinfo; +}; + +} + +#endif +#endif diff --git a/src/KOKKOS/compute_grid_kokkos.cpp b/src/KOKKOS/compute_grid_kokkos.cpp index 64c10d5db..4cd3472c0 100644 --- a/src/KOKKOS/compute_grid_kokkos.cpp +++ b/src/KOKKOS/compute_grid_kokkos.cpp @@ -38,10 +38,6 @@ enum{NUM,NRHO,NFRAC,MASS,MASSRHO,MASSFRAC, enum{COUNT,MASSSUM,MVX,MVY,MVZ,MVXSQ,MVYSQ,MVZSQ,MVSQ, ENGROT,ENGVIB,DOFROT,DOFVIB,CELLCOUNT,CELLMASS,LASTSIZE}; -// max # of quantities to accumulate for any user value - -#define MAXACCUMULATE 2 - /* ---------------------------------------------------------------------- */ ComputeGridKokkos::ComputeGridKokkos(SPARTA *sparta, int narg, char **arg) : diff --git a/src/KOKKOS/compute_isurf_grid_kokkos.cpp b/src/KOKKOS/compute_isurf_grid_kokkos.cpp index 84b97f862..e43338b09 100644 --- a/src/KOKKOS/compute_isurf_grid_kokkos.cpp +++ b/src/KOKKOS/compute_isurf_grid_kokkos.cpp @@ -107,6 +107,15 @@ void ComputeISurfGridKokkos::clear() // dispatches here virtually and resizes the device tally views, so they // are always current by the time tallying starts + // tallyinfo() compresses the tally list in place on the host and claims the + // host for it. A new cycle starts here: the device copy below is about to be + // zeroed and refilled by the tally kernels, so the two are deliberately + // parted and neither owes the other a copy. Without this the modify_device() + // in post_surf_tally() would meet the outstanding host claim and abort. + + k_tally2surf.clear_sync_state(); + k_array_surf_tally.clear_sync_state(); + Kokkos::deep_copy(d_array_surf_tally,0); Kokkos::deep_copy(d_surf2tally,-1); @@ -195,6 +204,14 @@ int ComputeISurfGridKokkos::tallyinfo(surfint *&ptr) tally2surf[istart] = tally2surf[iend]; } + // the compression above rewrote both arrays on the host. Claim it: the dense + // list is what every consumer reads, and an unclaimed write is discarded by + // the next sync_host() or by the realloc in init_normflux(), which resizes + // whichever side the counters call newer. + + k_tally2surf.modify_host(); + k_array_surf_tally.modify_host(); + return ntally; } diff --git a/src/KOKKOS/compute_ke_particle_kokkos.cpp b/src/KOKKOS/compute_ke_particle_kokkos.cpp index 66af19822..1b10447ea 100644 --- a/src/KOKKOS/compute_ke_particle_kokkos.cpp +++ b/src/KOKKOS/compute_ke_particle_kokkos.cpp @@ -45,6 +45,14 @@ ComputeKEParticleKokkos::~ComputeKEParticleKokkos() if (copymode) return; memoryKK->destroy_kokkos(k_vector_particle,vector_particle); vector_particle = NULL; + + // ke aliases vector_particle in the host implementation + // (compute_ke_particle.cpp:71 "vector_particle = ke"), and + // ~ComputeKEParticle() frees ke. Without clearing it here that base + // destructor frees the allocation just released above -- a double free + // at shutdown, after a run that produced entirely correct output + + ke = NULL; } /* ---------------------------------------------------------------------- */ diff --git a/src/KOKKOS/compute_lambda_grid_kokkos.cpp b/src/KOKKOS/compute_lambda_grid_kokkos.cpp index 2f2243743..20db5c4eb 100644 --- a/src/KOKKOS/compute_lambda_grid_kokkos.cpp +++ b/src/KOKKOS/compute_lambda_grid_kokkos.cpp @@ -199,6 +199,9 @@ void ComputeLambdaGridKokkos::compute_per_grid_kokkos() if (!fix->kokkos_flag) error->all(FLERR,"Cannot (yet) use non-Kokkos fixes with compute lambda/grid/kk"); KokkosBase* fKKBase = dynamic_cast(fix); + // a fix keeps its per-grid output between invocations and grid migration + // can leave it current on the host alone, so ask for the device copy + if (fKKBase) fKKBase->sync_pergrid_device_kokkos(); const int k = umap[m][0]; if (j == 0) { @@ -245,6 +248,9 @@ void ComputeLambdaGridKokkos::compute_per_grid_kokkos() if (!ftemp->kokkos_flag) error->all(FLERR,"Cannot (yet) use non-Kokkos fixes with compute lambda/grid/kk"); KokkosBase* ftempKKBase = dynamic_cast(ftemp); + // a fix keeps its per-grid output between invocations and grid migration + // can leave it current on the host alone, so ask for the device copy + if (ftempKKBase) ftempKKBase->sync_pergrid_device_kokkos(); if (tempindex == 0) { DAT::t_float_1d ft_vector_grid = ftempKKBase->d_vector_grid; diff --git a/src/KOKKOS/compute_react_boundary_kokkos.cpp b/src/KOKKOS/compute_react_boundary_kokkos.cpp new file mode 100644 index 000000000..5005b9a56 --- /dev/null +++ b/src/KOKKOS/compute_react_boundary_kokkos.cpp @@ -0,0 +1,146 @@ +/* ---------------------------------------------------------------------- + SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer + http://sparta.github.io + Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov + Sandia National Laboratories + + Copyright (2014) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level SPARTA directory. +------------------------------------------------------------------------- */ + +#include "string.h" +#include "compute_react_boundary_kokkos.h" +#include "surf.h" +#include "surf_react.h" +#include "update.h" +#include "domain.h" +#include "comm.h" +#include "memory_kokkos.h" +#include "error.h" +#include "kokkos.h" + +using namespace SPARTA_NS; + +/* ---------------------------------------------------------------------- */ + +ComputeReactBoundaryKokkos:: +ComputeReactBoundaryKokkos(SPARTA *sparta, int narg, char **arg) : + ComputeReactBoundary(sparta, narg, arg) +{ + kokkos_flag = 1; + + memory->destroy(myarray); + memoryKK->create_kokkos(k_myarray,myarray,size_array_rows,size_array_cols, + "react/boundary:myarray"); + d_myarray = k_myarray.view_device(); + + // reaction2col never changes after construction, so flatten it once + // rpflag = 0 leaves it unallocated on the host; the kernel does not read + // it in that case, but give it one element so the view is always valid + + if (rpflag) { + DAT::tdual_int_2d k_r2c("react/boundary:reaction2col", + surf->sr[isr]->nlist,ntotal); + for (int i = 0; i < surf->sr[isr]->nlist; i++) + for (int j = 0; j < ntotal; j++) + k_r2c.view_host()(i,j) = reaction2col[i][j]; + k_r2c.modify_host(); + k_r2c.sync_device(); + d_reaction2col = k_r2c.view_device(); + } else { + d_reaction2col = DAT::t_int_2d("react/boundary:reaction2col",1,1); + } + + d_surf_react = DAT::t_int_1d("react/boundary:surf_react",6); +} + +/* ---------------------------------------------------------------------- */ + +ComputeReactBoundaryKokkos::ComputeReactBoundaryKokkos(SPARTA *sparta) : + ComputeReactBoundary(sparta) +{ + copy = 1; +} + +/* ---------------------------------------------------------------------- */ + +ComputeReactBoundaryKokkos::~ComputeReactBoundaryKokkos() +{ + if (copy || copymode) return; + + memoryKK->destroy_kokkos(k_myarray,myarray); + myarray = NULL; +} + +/* ---------------------------------------------------------------------- */ + +void ComputeReactBoundaryKokkos::init() +{ + if (!domain->surfreactany && comm->me == 0) + error->warning(FLERR,"Using compute react/boundary " + "when no box faces are assigned a reaction model"); + + clear(); +} + +/* ---------------------------------------------------------------------- + called by Update at the start of any timestep boundary tallying is done +------------------------------------------------------------------------- */ + +void ComputeReactBoundaryKokkos::clear() +{ + Kokkos::deep_copy(d_myarray,0.0); +} + +/* ---------------------------------------------------------------------- + called by UpdateKokkos before the move kernel +------------------------------------------------------------------------- */ + +void ComputeReactBoundaryKokkos::pre_boundary_tally() +{ + // domain->surf_react is a small host array indexed by box face, and + // bound_modify can change it between runs, so refresh it each step + + auto h_surf_react = Kokkos::create_mirror_view(d_surf_react); + for (int i = 0; i < 6; i++) h_surf_react(i) = domain->surf_react[i]; + Kokkos::deep_copy(d_surf_react,h_surf_react); + + need_dup = sparta->kokkos->need_dup(); + if (need_dup) + dup_myarray = Kokkos::Experimental::create_scatter_view(d_myarray); + else + ndup_myarray = Kokkos::Experimental::create_scatter_view(d_myarray); +} + +/* ---------------------------------------------------------------------- + called by UpdateKokkos after the move kernel +------------------------------------------------------------------------- */ + +void ComputeReactBoundaryKokkos::post_boundary_tally() +{ + if (need_dup) { + Kokkos::Experimental::contribute(d_myarray, dup_myarray); + dup_myarray = {}; // free duplicated memory + } +} + +/* ---------------------------------------------------------------------- + sum tallies across processors, as the host version does +------------------------------------------------------------------------- */ + +void ComputeReactBoundaryKokkos::compute_array() +{ + invoked_array = update->ntimestep; + + // the reduction is a host MPI call, so bring the device tallies back first + + k_myarray.modify_device(); + k_myarray.sync_host(); + + MPI_Allreduce(&myarray[0][0],&array[0][0],nrow*ntotal, + MPI_DOUBLE,MPI_SUM,world); +} diff --git a/src/KOKKOS/compute_react_boundary_kokkos.h b/src/KOKKOS/compute_react_boundary_kokkos.h new file mode 100644 index 000000000..9be68febb --- /dev/null +++ b/src/KOKKOS/compute_react_boundary_kokkos.h @@ -0,0 +1,96 @@ +/* ---------------------------------------------------------------------- + SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer + http://sparta.github.io + Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov + Sandia National Laboratories + + Copyright (2014) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level SPARTA directory. +------------------------------------------------------------------------- */ + +#ifdef COMPUTE_CLASS + +ComputeStyle(react/boundary/kk,ComputeReactBoundaryKokkos) + +#else + +#ifndef SPARTA_REACT_BOUNDARY_KOKKOS_H +#define SPARTA_REACT_BOUNDARY_KOKKOS_H + +#include "compute_react_boundary.h" +#include "kokkos_base.h" +#include "kokkos_type.h" + +namespace SPARTA_NS { + +class ComputeReactBoundaryKokkos : public ComputeReactBoundary, public KokkosBase { + public: + ComputeReactBoundaryKokkos(class SPARTA *, int, char **); + ComputeReactBoundaryKokkos(class SPARTA *); + ~ComputeReactBoundaryKokkos() override; + + void init() override; + void compute_array() override; + void clear() override; + + // called by UpdateKokkos around the move kernel + + void pre_boundary_tally(); + void post_boundary_tally(); + + /* ---------------------------------------------------------------------- + tally a surface reaction on box face iface + mirrors ComputeReactBoundary::boundary_tally() + (compute_react_boundary.cpp:139-166) exactly + norm is unused here; it is in the signature so UpdateKokkos can dispatch + every boundary tally compute through one call + ------------------------------------------------------------------------- */ + + template + KOKKOS_INLINE_FUNCTION + void boundary_tally_kk(double /*dtremain*/, int iface, int /*istyle*/, + int reaction, Particle::OnePart * /*iorig*/, + Particle::OnePart * /*ip*/, Particle::OnePart * /*jp*/, + const double * /*norm*/) const + { + // skip if no reaction + + if (reaction == 0) return; + reaction--; + + // skip if this face's reaction model is not a match + + if (d_surf_react[iface] != isr) return; + + auto v_myarray = ScatterViewHelper::value,decltype(dup_myarray),decltype(ndup_myarray)>::get(dup_myarray,ndup_myarray); + auto a_myarray = v_myarray.template access::value>(); + + // for rpflag, tally each column whose reaction2col entry is set + // for rpflag = 0, tally the reaction directly + + if (rpflag) { + for (int i = 0; i < ntotal; i++) + if (d_reaction2col(reaction,i)) a_myarray(iface,i) += 1.0; + } else a_myarray(iface,reaction) += 1.0; + } + + private: + DAT::tdual_float_2d_lr k_myarray; // local accumulator array + DAT::t_float_2d_lr d_myarray; + + DAT::t_int_1d d_surf_react; // per box face: its surf react model + DAT::t_int_2d d_reaction2col; // 1 if ireaction tallies into icol + + int need_dup; + Kokkos::Experimental::ScatterView dup_myarray; + Kokkos::Experimental::ScatterView ndup_myarray; +}; + +} + +#endif +#endif diff --git a/src/KOKKOS/compute_react_surf_kokkos.cpp b/src/KOKKOS/compute_react_surf_kokkos.cpp index 5dbb8b416..7fe49d4ba 100644 --- a/src/KOKKOS/compute_react_surf_kokkos.cpp +++ b/src/KOKKOS/compute_react_surf_kokkos.cpp @@ -174,6 +174,13 @@ int ComputeReactSurfKokkos::tallyinfo(surfint *&ptr) tally2surf[istart] = tally2surf[iend]; } + // see ComputeSurfKokkos::tallyinfo(): the compression rewrites the host side + // in place and the caller consumes it immediately, so the two sides are + // deliberately apart until the next clear() + + k_tally2surf.clear_sync_state(); + k_array_surf_tally.clear_sync_state(); + return ntally; } diff --git a/src/KOKKOS/compute_reduce_kokkos.cpp b/src/KOKKOS/compute_reduce_kokkos.cpp new file mode 100644 index 000000000..79bee4c15 --- /dev/null +++ b/src/KOKKOS/compute_reduce_kokkos.cpp @@ -0,0 +1,875 @@ +/* ---------------------------------------------------------------------- + SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer + http://sparta.github.io + Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov + Sandia National Laboratories + + Copyright (2014) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level SPARTA directory. +------------------------------------------------------------------------- */ + +#include "mpi.h" +#include "string.h" +#include "stdlib.h" +#include "compute_reduce_kokkos.h" +#include "update.h" +#include "domain.h" +#include "particle_kokkos.h" +#include "mixture.h" +#include "grid_kokkos.h" +#include "surf_kokkos.h" +#include "modify.h" +#include "fix.h" +#include "compute.h" +#include "input.h" +#include "variable.h" +#include "memory_kokkos.h" +#include "error.h" +#include "sparta_masks.h" +#include "kokkos.h" + +using namespace SPARTA_NS; + +// these must stay in lock step with the enums in compute_reduce.cpp + +enum{SUM,SUMSQ,MINN,MAXX,AVE,AVESQ,SUMAREA,AVEAREA}; +enum{X,V,KE,EROT,EVIB,COMPUTE,FIX,VARIABLE,PCUSTOM,GCUSTOM,SCUSTOM}; +enum{PARTICLE,GRID,SURF}; + +enum{INT,DOUBLE}; // several files + +#define INVOKED_PER_PARTICLE 8 +#define INVOKED_PER_GRID 16 +#define INVOKED_PER_SURF 32 + +#define BIG 1.0e20 + +/* ---------------------------------------------------------------------- */ + +ComputeReduceKokkos::ComputeReduceKokkos(SPARTA *sparta, int narg, char **arg) : + ComputeReduce(sparta, narg, arg) +{ + kokkos_flag = 1; + + nelements = maxelements = 0; +} + +/* ---------------------------------------------------------------------- */ + +ComputeReduceKokkos::~ComputeReduceKokkos() +{ + // no explicit deallocation: the scratch Kokkos views free themselves. + // deliberately NOT guarded with "if (copymode) return;": nothing here is + // ever copied into a Kokkos functor (see the comment in the header), and + // a guard would only paper over the fact that the base class destructor + // would double-free anyway if a copy were ever made +} + +/* ---------------------------------------------------------------------- */ + +void ComputeReduceKokkos::init() +{ + // everything this style needs is set up by the base class: + // s2g (mixture map), gridgroupbit, smasks/areasurf/area_total for SURF, + // and value2index for every input. the device-side subset test reads a + // fresh copy of s2g in build_include(), so nothing extra is recorded here + + ComputeReduce::init(); +} + +/* ---------------------------------------------------------------------- */ + +double ComputeReduceKokkos::compute_scalar() +{ + if (sparta->kokkos->prewrap) return ComputeReduce::compute_scalar(); + + invoked_scalar = update->ntimestep; + + double one = compute_one_kokkos(0,-1); + + // MPI reduction is identical to ComputeReduce::compute_scalar() + + if (mode == SUM || mode == SUMSQ || mode == SUMAREA) { + MPI_Allreduce(&one,&scalar,1,MPI_DOUBLE,MPI_SUM,world); + } else if (mode == MINN) { + MPI_Allreduce(&one,&scalar,1,MPI_DOUBLE,MPI_MIN,world); + } else if (mode == MAXX) { + MPI_Allreduce(&one,&scalar,1,MPI_DOUBLE,MPI_MAX,world); + } else if (mode == AVE || mode == AVESQ) { + MPI_Allreduce(&one,&scalar,1,MPI_DOUBLE,MPI_SUM,world); + bigint n = count_included_kokkos(); + if (n) scalar /= n; + } else if (mode == AVEAREA) { + MPI_Allreduce(&one,&scalar,1,MPI_DOUBLE,MPI_SUM,world); + if (area_total > 0.0) scalar /= area_total; + } + + return scalar; +} + +/* ---------------------------------------------------------------------- */ + +void ComputeReduceKokkos::compute_vector() +{ + if (sparta->kokkos->prewrap) { + ComputeReduce::compute_vector(); + return; + } + + invoked_vector = update->ntimestep; + + for (int m = 0; m < nvalues; m++) + if (!replace || replace[m] < 0) { + onevec[m] = compute_one_kokkos(m,-1); + indices[m] = index; + } + + // MPI reduction is identical to ComputeReduce::compute_vector() + + if (mode == SUM || mode == SUMSQ || mode == SUMAREA) { + for (int m = 0; m < nvalues; m++) + MPI_Allreduce(&onevec[m],&vector[m],1,MPI_DOUBLE,MPI_SUM,world); + + } else if (mode == MINN) { + if (!replace) { + for (int m = 0; m < nvalues; m++) + MPI_Allreduce(&onevec[m],&vector[m],1,MPI_DOUBLE,MPI_MIN,world); + + } else { + for (int m = 0; m < nvalues; m++) + if (replace[m] < 0) { + pairme.value = onevec[m]; + pairme.proc = me; + MPI_Allreduce(&pairme,&pairall,1,MPI_DOUBLE_INT,MPI_MINLOC,world); + vector[m] = pairall.value; + owner[m] = pairall.proc; + } + for (int m = 0; m < nvalues; m++) + if (replace[m] >= 0) { + if (me == owner[replace[m]]) + vector[m] = compute_one_kokkos(m,indices[replace[m]]); + MPI_Bcast(&vector[m],1,MPI_DOUBLE,owner[replace[m]],world); + } + } + + } else if (mode == MAXX) { + if (!replace) { + for (int m = 0; m < nvalues; m++) + MPI_Allreduce(&onevec[m],&vector[m],1,MPI_DOUBLE,MPI_MAX,world); + + } else { + for (int m = 0; m < nvalues; m++) + if (replace[m] < 0) { + pairme.value = onevec[m]; + pairme.proc = me; + MPI_Allreduce(&pairme,&pairall,1,MPI_DOUBLE_INT,MPI_MAXLOC,world); + vector[m] = pairall.value; + owner[m] = pairall.proc; + } + for (int m = 0; m < nvalues; m++) + if (replace[m] >= 0) { + if (me == owner[replace[m]]) + vector[m] = compute_one_kokkos(m,indices[replace[m]]); + MPI_Bcast(&vector[m],1,MPI_DOUBLE,owner[replace[m]],world); + } + } + + } else if (mode == AVE || mode == AVESQ) { + bigint n = count_included_kokkos(); + for (int m = 0; m < nvalues; m++) { + MPI_Allreduce(&onevec[m],&vector[m],1,MPI_DOUBLE,MPI_SUM,world); + if (n) vector[m] /= n; + } + + } else if (mode == AVEAREA) { + for (int m = 0; m < nvalues; m++) { + MPI_Allreduce(&onevec[m],&vector[m],1,MPI_DOUBLE,MPI_SUM,world); + if (area_total > 0.0) vector[m] /= area_total; + } + } +} + +/* ---------------------------------------------------------------------- + bring the host-side arrays that ComputeReduce::compute_one() reads back + up to date before delegating one input to it. + in a Kokkos run the authoritative copy of particles / cinfo / custom + attributes is the device one, so a host fall-back that skipped this + would silently reduce stale data (the subset test in particular reads + particle->particles[i].ispecies and grid->cinfo[i].mask directly) +------------------------------------------------------------------------- */ + +void ComputeReduceKokkos::sync_host_for_fallback(int m) +{ + if (flavor[m] == PARTICLE) { + ParticleKokkos *particle_kk = (ParticleKokkos *) particle; + particle_kk->sync(Host,PARTICLE_MASK|SPECIES_MASK|CUSTOM_MASK); + } else if (flavor[m] == GRID) { + GridKokkos *grid_kk = (GridKokkos *) grid; + grid_kk->sync(Host,CINFO_MASK|CUSTOM_MASK); + } else if (surf->exist) { + SurfKokkos *surf_kk = (SurfKokkos *) surf; + surf_kk->sync(Host,LINE_MASK|TRI_MASK|MYLINE_MASK|MYTRI_MASK|CUSTOM_MASK); + } +} + +/* ---------------------------------------------------------------------- + calculate reduced value for one input M and return it + same contract as ComputeReduce::compute_one(): + flag = -1: sum/min/max/ave all values, set index for MIN/MAX + flag >= 0: simply return the value of element flag + falls back to the host implementation whenever input M has no + device-resident source (see setup_values) +------------------------------------------------------------------------- */ + +double ComputeReduceKokkos::compute_one_kokkos(int m, int flag) +{ + index = -1; + + // SURF inputs stay on the host. KokkosBase exposes no d_vector_surf or + // d_array_surf, the surf->nown ownership decomposition and the smasks[] + // /areasurf[] weights that ComputeReduce::init() builds live only on the + // host, and SUMAREA/AVEAREA are per-surf by construction. Reducing on + // the host is also trivially bit-for-bit identical + + if (flavor[m] == SURF) { + sync_host_for_fallback(m); + return ComputeReduce::compute_one(m,flag); + } + + // gather element values for input m into d_values; 0 means no device source + + if (!setup_values(m)) { + sync_host_for_fallback(m); + return ComputeReduce::compute_one(m,flag); + } + + // flag >= 0: return the single element, ignoring the subset mask, + // exactly as the host does + + if (flag >= 0) { + double one = 0.0; + if (flag < nelements) { + + // the scalar handed to deep_copy(value,View) is a non-deduced + // parameter, so it has to be spelled with the view's own value type, + // SPARTA_FLOAT, not double, or template deduction fails + + SPARTA_FLOAT tmp = 0.0; + Kokkos::deep_copy(tmp,Kokkos::subview(d_values,flag)); + one = tmp; + } + return one; + } + + return reduce_values(); +} + +/* ---------------------------------------------------------------------- + set up the device source for input m and gather it into d_values + return 1 if the gather was done on the device, 0 to fall back to the host + an early 0 return must not have invoked the source compute/fix, so that + ComputeReduce::compute_one() can still invoke it itself +------------------------------------------------------------------------- */ + +int ComputeReduceKokkos::setup_values(int m) +{ + const int vidx = value2index[m]; + const int aidx = argindex[m]; + const int acol = aidx - 1; + + // particle-style / grid-style variables are evaluated by the host Variable + // class, there is no device evaluator. Copying the host result to the + // device only to reduce it there would buy nothing and would give up the + // host's summation order, so reduce on the host instead. + // decided first, so that nothing at all has been touched on this path + + if (which[m] == VARIABLE) return 0; + + // explicit per-particle attributes: always resident on the device + + if (which[m] == X || which[m] == V || which[m] == KE || + which[m] == EROT || which[m] == EVIB) { + + ParticleKokkos *particle_kk = (ParticleKokkos *) particle; + particle_kk->sync(Device,PARTICLE_MASK|SPECIES_MASK); + auto l_particles = particle_kk->k_particles.view_device(); + auto l_species = particle_kk->k_species.view_device(); + + build_include(m); + if (nelements == 0) return 1; + + auto l_values = d_values; + const int j = aidx; + const double mvv2e = update->mvv2e; + + if (which[m] == X) { + Kokkos::parallel_for(Kokkos::RangePolicy(0,nelements), + KOKKOS_LAMBDA(const int i) { + l_values(i) = l_particles(i).x[j]; + }); + + } else if (which[m] == V) { + Kokkos::parallel_for(Kokkos::RangePolicy(0,nelements), + KOKKOS_LAMBDA(const int i) { + l_values(i) = l_particles(i).v[j]; + }); + + } else if (which[m] == KE) { + + // expression order is copied verbatim from ComputeReduce::compute_one() + // so the per-particle value is bit-identical to the host's + + Kokkos::parallel_for(Kokkos::RangePolicy(0,nelements), + KOKKOS_LAMBDA(const int i) { + const double *v = l_particles(i).v; + l_values(i) = mvv2e * 0.5 * l_species(l_particles(i).ispecies).mass * + (v[0]*v[0] + v[1]*v[1] + v[2]*v[2]); + }); + + } else if (which[m] == EROT) { + Kokkos::parallel_for(Kokkos::RangePolicy(0,nelements), + KOKKOS_LAMBDA(const int i) { + l_values(i) = l_particles(i).erot; + }); + + } else { + Kokkos::parallel_for(Kokkos::RangePolicy(0,nelements), + KOKKOS_LAMBDA(const int i) { + l_values(i) = l_particles(i).evib; + }); + } + + return 1; + + // per-particle or per-grid output of another compute + + } else if (which[m] == COMPUTE) { + Compute *c = modify->compute[vidx]; + + // a non-Kokkos compute, or one that does not derive from KokkosBase, + // has no device output at all. neither test invokes anything + + if (!c->kokkos_flag) return 0; + KokkosBase *ckk = dynamic_cast(c); + if (!ckk) return 0; + + DAT::t_float_1d_strided d_src; + + if (flavor[m] == PARTICLE) { + + // KokkosBase declares no compute_per_particle_kokkos() entry point, so + // the compute is invoked through the normal Compute API. The /kk + // per-particle styles dispatch compute_per_particle() to their own + // device kernel whenever Kokkos is past its prewrap phase + + if (!(c->invoked_flag & INVOKED_PER_PARTICLE)) { + c->compute_per_particle(); + c->invoked_flag |= INVOKED_PER_PARTICLE; + } + + // CAVEAT: if the compute filled only its host vector_particle/ + // array_particle we fall back, and ComputeReduce::compute_one() will + // see invoked_flag already set and read those host arrays without + // re-invoking. That is correct only for a /kk style whose + // compute_per_particle() really did leave the HOST arrays current + // (e.g. one that just calls its non-Kokkos base). A style that + // computes on the device and does not publish d_vector_particle / + // d_array_particle would be reduced from stale host data here. No + // such style exists today (compute ke/particle/kk publishes + // d_vector_particle, fix field/particle/kk publishes + // d_array_particle) and KokkosBase gives us no way to force a + // sync_host() on someone else's DualView, so this is a documented + // limitation rather than a fixed problem + + if (aidx == 0) { + if (!ckk->d_vector_particle.data()) return 0; + if ((int) ckk->d_vector_particle.extent(0) < particle->nlocal) return 0; + d_src = ckk->d_vector_particle; + } else { + if (!ckk->d_array_particle.data()) return 0; + if ((int) ckk->d_array_particle.extent(0) < particle->nlocal || + (int) ckk->d_array_particle.extent(1) <= acol) return 0; + d_src = Kokkos::subview(ckk->d_array_particle,Kokkos::ALL(),acol); + } + + } else { // GRID + + // post_process_isurf_grid() has no device counterpart, so this whole + // input is reduced on the host. Checked before the invocation, so + // the host path is free to invoke the compute itself + + if (c->post_process_isurf_grid_flag) return 0; + + // canonical "consume another Kokkos compute's per-grid device output" + // sequence, same as FixAveGridKokkos::end_of_step() and + // FixAveHistoKokkos::end_of_step() + + if (!(c->invoked_flag & INVOKED_PER_GRID)) { + ckk->compute_per_grid_kokkos(); + c->invoked_flag |= INVOKED_PER_GRID; + } + + // must run for every input, not just the first: the column selects + // which post-processed quantity lands in d_vector_grid. passing + // null views is the device spelling of the host's + // post_process_grid(aidx,1,NULL,NULL,NULL,1) + + if (c->post_process_grid_flag) + ckk->post_process_grid_kokkos(aidx,1,DAT::t_float_2d_lr(),NULL, + DAT::t_float_1d_strided()); + + // a post-processing compute writes its answer to d_vector_grid + // regardless of aidx, matching the host's cvec/carray choice + + if (aidx == 0 || c->post_process_grid_flag) { + if (!ckk->d_vector_grid.data()) return 0; + if ((int) ckk->d_vector_grid.extent(0) < grid->nlocal) return 0; + d_src = ckk->d_vector_grid; + } else { + if (!ckk->d_array_grid.data()) return 0; + if ((int) ckk->d_array_grid.extent(0) < grid->nlocal || + (int) ckk->d_array_grid.extent(1) <= acol) return 0; + d_src = Kokkos::subview(ckk->d_array_grid,Kokkos::ALL(),acol); + } + } + + build_include(m); + gather_float(d_src); + return 1; + + // per-particle or per-grid output of a fix + // fixes are not invoked here, they are guaranteed to be up to date + + } else if (which[m] == FIX) { + Fix *fix = modify->fix[vidx]; + + // preserve the host's timestep compatibility check, before any fallback + + if (flavor[m] == PARTICLE) { + if (update->ntimestep % fix->per_particle_freq) + error->all(FLERR,"Fix used in compute reduce not " + "computed at compatible time"); + } else { + if (update->ntimestep % fix->per_grid_freq) + error->all(FLERR,"Fix used in compute reduce not " + "computed at compatible time"); + } + + if (!fix->kokkos_flag) return 0; + KokkosBase *fkk = dynamic_cast(fix); + if (!fkk) return 0; + + DAT::t_float_1d_strided d_src; + + if (flavor[m] == PARTICLE) { + if (aidx == 0) { + if (!fkk->d_vector_particle.data()) return 0; + if ((int) fkk->d_vector_particle.extent(0) < particle->nlocal) return 0; + d_src = fkk->d_vector_particle; + } else { + if (!fkk->d_array_particle.data()) return 0; + if ((int) fkk->d_array_particle.extent(0) < particle->nlocal || + (int) fkk->d_array_particle.extent(1) <= acol) return 0; + d_src = Kokkos::subview(fkk->d_array_particle,Kokkos::ALL(),acol); + } + } else { + if (aidx == 0) { + if (!fkk->d_vector_grid.data()) return 0; + if ((int) fkk->d_vector_grid.extent(0) < grid->nlocal) return 0; + d_src = fkk->d_vector_grid; + } else { + if (!fkk->d_array_grid.data()) return 0; + if ((int) fkk->d_array_grid.extent(0) < grid->nlocal || + (int) fkk->d_array_grid.extent(1) <= acol) return 0; + d_src = Kokkos::subview(fkk->d_array_grid,Kokkos::ALL(),acol); + } + } + + build_include(m); + gather_float(d_src); + return 1; + + // per-particle custom attribute + // index the plain host ewhich[]/etype[] arrays, exactly as the non-Kokkos + // path does, and take the device half of the matching DualView + + } else if (which[m] == PCUSTOM) { + ParticleKokkos *particle_kk = (ParticleKokkos *) particle; + particle_kk->sync(Device,CUSTOM_MASK); + + const int ew = particle->ewhich[vidx]; + if (ew < 0) return 0; + + if (particle->etype[vidx] == INT) { + if (aidx == 0) { + if (ew >= (int) particle_kk->k_eivec.view_host().extent(0)) return 0; + auto d_src = particle_kk->k_eivec.view_host()[ew].k_view.view_device(); + if (!d_src.data()) return 0; + if ((int) d_src.extent(0) < particle->nlocal) return 0; + build_include(m); + gather_int_vec(d_src); + } else { + if (ew >= (int) particle_kk->k_eiarray.view_host().extent(0)) return 0; + auto d_src = particle_kk->k_eiarray.view_host()[ew].k_view.view_device(); + if (!d_src.data()) return 0; + if ((int) d_src.extent(0) < particle->nlocal || + (int) d_src.extent(1) <= acol) return 0; + build_include(m); + gather_int_array(d_src,acol); + } + } else { + if (aidx == 0) { + if (ew >= (int) particle_kk->k_edvec.view_host().extent(0)) return 0; + auto d_src = particle_kk->k_edvec.view_host()[ew].k_view.view_device(); + if (!d_src.data()) return 0; + if ((int) d_src.extent(0) < particle->nlocal) return 0; + build_include(m); + gather_float(d_src); + } else { + if (ew >= (int) particle_kk->k_edarray.view_host().extent(0)) return 0; + auto d_src = particle_kk->k_edarray.view_host()[ew].k_view.view_device(); + if (!d_src.data()) return 0; + if ((int) d_src.extent(0) < particle->nlocal || + (int) d_src.extent(1) <= acol) return 0; + build_include(m); + gather_float(Kokkos::subview(d_src,Kokkos::ALL(),acol)); + } + } + + return 1; + + // per-grid custom attribute + + } else if (which[m] == GCUSTOM) { + GridKokkos *grid_kk = (GridKokkos *) grid; + grid_kk->sync(Device,CUSTOM_MASK); + + const int ew = grid->ewhich[vidx]; + if (ew < 0) return 0; + + if (grid->etype[vidx] == INT) { + if (aidx == 0) { + if (ew >= (int) grid_kk->k_eivec.view_host().extent(0)) return 0; + auto d_src = grid_kk->k_eivec.view_host()[ew].k_view.view_device(); + if (!d_src.data()) return 0; + if ((int) d_src.extent(0) < grid->nlocal) return 0; + build_include(m); + gather_int_vec(d_src); + } else { + if (ew >= (int) grid_kk->k_eiarray.view_host().extent(0)) return 0; + auto d_src = grid_kk->k_eiarray.view_host()[ew].k_view.view_device(); + if (!d_src.data()) return 0; + if ((int) d_src.extent(0) < grid->nlocal || + (int) d_src.extent(1) <= acol) return 0; + build_include(m); + gather_int_array(d_src,acol); + } + } else { + if (aidx == 0) { + if (ew >= (int) grid_kk->k_edvec.view_host().extent(0)) return 0; + auto d_src = grid_kk->k_edvec.view_host()[ew].k_view.view_device(); + if (!d_src.data()) return 0; + if ((int) d_src.extent(0) < grid->nlocal) return 0; + build_include(m); + gather_float(d_src); + } else { + if (ew >= (int) grid_kk->k_edarray.view_host().extent(0)) return 0; + auto d_src = grid_kk->k_edarray.view_host()[ew].k_view.view_device(); + if (!d_src.data()) return 0; + if ((int) d_src.extent(0) < grid->nlocal || + (int) d_src.extent(1) <= acol) return 0; + build_include(m); + gather_float(Kokkos::subview(d_src,Kokkos::ALL(),acol)); + } + } + + return 1; + } + + // SCUSTOM and anything else: host. (SCUSTOM is flavor SURF and has + // already been intercepted by compute_one_kokkos) + + return 0; +} + +/* ---------------------------------------------------------------------- + copy the selected source into the contiguous d_values scratch view + build_include() must have run first: it sets nelements and (re)sizes + d_values, so taking the local copy afterwards is mandatory +------------------------------------------------------------------------- */ + +void ComputeReduceKokkos::gather_float(DAT::t_float_1d_strided d_src) +{ + if (nelements == 0) return; + auto l_values = d_values; + auto l_src = d_src; + Kokkos::parallel_for(Kokkos::RangePolicy(0,nelements), + KOKKOS_LAMBDA(const int i) { + l_values(i) = l_src(i); + }); +} + +/* ---------------------------------------------------------------------- */ + +void ComputeReduceKokkos::gather_int_vec(DAT::t_int_1d d_src) +{ + if (nelements == 0) return; + auto l_values = d_values; + auto l_src = d_src; + Kokkos::parallel_for(Kokkos::RangePolicy(0,nelements), + KOKKOS_LAMBDA(const int i) { + l_values(i) = l_src(i); + }); +} + +/* ---------------------------------------------------------------------- */ + +void ComputeReduceKokkos::gather_int_array(DAT::t_int_2d_lr d_src, int acol) +{ + if (nelements == 0) return; + auto l_values = d_values; + auto l_src = d_src; + const int j = acol; + Kokkos::parallel_for(Kokkos::RangePolicy(0,nelements), + KOKKOS_LAMBDA(const int i) { + l_values(i) = l_src(i,j); + }); +} + +/* ---------------------------------------------------------------------- + set nelements, size the scratch views, and fill d_include with the subset + membership test. the test only depends on flavor and subsetID, not on + which input it is +------------------------------------------------------------------------- */ + +void ComputeReduceKokkos::build_include(int m) +{ + if (flavor[m] == PARTICLE) nelements = particle->nlocal; + else nelements = grid->nlocal; + + grow_scratch(nelements); + if (nelements == 0) return; + + // no subset: every element participates + + if (!subsetID) { + Kokkos::deep_copy(Kokkos::subview(d_include, + Kokkos::make_pair(0,nelements)),1); + return; + } + + auto l_include = d_include; + + if (flavor[m] == PARTICLE) { + ParticleKokkos *particle_kk = (ParticleKokkos *) particle; + particle_kk->sync(Device,PARTICLE_MASK); + auto l_particles = particle_kk->k_particles.view_device(); + + // refresh the device copy of the base class's s2g every time. + // ParticleKokkos::k_species2group is only built once, inside + // wrap_kokkos(), so it can be both stale and too short if species or + // mixtures change after the first setup -- indexing it on the device + // would then read out of bounds. s2g always has particle->nspecies + // entries (Mixture::init) and is the exact array the non-Kokkos path + // reads, so copying it is both cheap and definitionally in agreement + + const int nsp = particle->nspecies; + if ((int) k_s2g.view_host().extent(0) != nsp) + MemKK::realloc_kokkos(k_s2g,"reduce/kk:s2g",nsp); + auto hv_s2g = k_s2g.view_host(); + for (int i = 0; i < nsp; i++) hv_s2g(i) = s2g[i]; + k_s2g.modify_host(); + k_s2g.sync_device(); + + auto l_s2g = k_s2g.view_device(); + Kokkos::parallel_for(Kokkos::RangePolicy(0,nelements), + KOKKOS_LAMBDA(const int i) { + l_include(i) = (l_s2g(l_particles(i).ispecies) >= 0) ? 1 : 0; + }); + + } else { + GridKokkos *grid_kk = (GridKokkos *) grid; + grid_kk->sync(Device,CINFO_MASK); + auto l_cinfo = grid_kk->k_cinfo.view_device(); + const int l_groupbit = gridgroupbit; + Kokkos::parallel_for(Kokkos::RangePolicy(0,nelements), + KOKKOS_LAMBDA(const int i) { + l_include(i) = (l_cinfo(i).mask & l_groupbit) ? 1 : 0; + }); + } +} + +/* ---------------------------------------------------------------------- + reduce d_values over the included elements according to mode + also sets index for MIN/MAX when the replace option is in use +------------------------------------------------------------------------- */ + +double ComputeReduceKokkos::reduce_values() +{ + double one = 0.0; + if (mode == MINN) one = BIG; + else if (mode == MAXX) one = -BIG; + + if (nelements == 0) return one; + + const int n = nelements; + +#ifdef SPARTA_KOKKOS_EXACT + + // SPARTA_KOKKOS_EXACT asks the Kokkos package to reproduce the non-Kokkos + // result bit for bit so the KOKKOS build can be regression tested + // against the existing gold-standard logs. Floating point addition is + // not associative, so no tree reduction can be relied on to reproduce + // ComputeReduce::compute_one()'s left-to-right accumulation over + // ascending element index. Bring the gathered values back to the host + // and run the base class's own combine() over them in exactly that + // order. MIN/MAX would in principle survive a tree reduction, but they + // go through here as well: combine() then also sets "index" the way the + // host sets it, including its first-index-wins tie break and its + // treatment of +/-0.0, so there is nothing left to argue about + + // local mirrors on purpose: View::HostMirror is deprecation-gated in + // recent Kokkos, and this path only runs in a SPARTA_KOKKOS_EXACT + // regression build where the extra allocation does not matter + + auto h_values = Kokkos::create_mirror_view(d_values); + auto h_include = Kokkos::create_mirror_view(d_include); + Kokkos::deep_copy(h_values,d_values); + Kokkos::deep_copy(h_include,d_include); + + for (int i = 0; i < n; i++) { + if (!h_include(i)) continue; + combine(one,h_values(i),i); + } + return one; + +#else + + auto l_values = d_values; + auto l_include = d_include; + + if (mode == SUM || mode == AVE || mode == SUMAREA || mode == AVEAREA) { + + // NOTE: summation order is the Kokkos backend's, not the host's. For + // SUM/AVE that difference is not observable except in the last bits; + // see the SPARTA_KOKKOS_EXACT branch above for the bit-exact path + + Kokkos::parallel_reduce(Kokkos::RangePolicy(0,n), + KOKKOS_LAMBDA(const int i, double &lsum) { + if (l_include(i)) lsum += l_values(i); + },one); + + } else if (mode == SUMSQ || mode == AVESQ) { + Kokkos::parallel_reduce(Kokkos::RangePolicy(0,n), + KOKKOS_LAMBDA(const int i, double &lsum) { + if (l_include(i)) lsum += l_values(i)*l_values(i); + },one); + + } else if (mode == MINN) { + + // min/max are order independent, so the device result matches the host. + // Kokkos ignores the value a reducer is constructed with and starts + // from its own identity (+/- DBL_MAX), so the result has to be folded + // back into the host's BIG seed rather than used directly. That also + // covers the empty / all-excluded case + + double vmin = BIG; + Kokkos::parallel_reduce(Kokkos::RangePolicy(0,n), + KOKKOS_LAMBDA(const int i, double &lmin) { + if (l_include(i) && l_values(i) < lmin) lmin = l_values(i); + },Kokkos::Min(vmin)); + if (vmin < one) one = vmin; + + } else if (mode == MAXX) { + double vmax = -BIG; + Kokkos::parallel_reduce(Kokkos::RangePolicy(0,n), + KOKKOS_LAMBDA(const int i, double &lmax) { + if (l_include(i) && l_values(i) > lmax) lmax = l_values(i); + },Kokkos::Max(vmax)); + if (vmax > one) one = vmax; + } + + // ComputeReduce::combine() also records the index of the winning element. + // It is only ever consumed by the "replace" option, which in turn is only + // legal for min/max, so the extra pass is skipped otherwise. + // The host's strict < / > leaves the FIRST winning index behind, so take + // the smallest index attaining the winning value. Kokkos::MinLoc cannot + // be used for this: its join only breaks a tie against the identity, so + // the index it returns depends on the thread decomposition. + // "seeded" reproduces the host leaving index = -1 when nothing ever beat + // its BIG / -BIG seed + + if (replace && (mode == MINN || mode == MAXX)) { + const double target = one; + const int seeded = (mode == MINN) ? (one < BIG) : (one > -BIG); + if (seeded) { + int iwin = n; + Kokkos::parallel_reduce(Kokkos::RangePolicy(0,n), + KOKKOS_LAMBDA(const int i, int &lidx) { + if (l_include(i) && l_values(i) == target && i < lidx) lidx = i; + },Kokkos::Min(iwin)); + if (iwin < n) index = iwin; + } + } + + return one; + +#endif +} + +/* ---------------------------------------------------------------------- + count elements included in the reduction, summed across procs + device version of ComputeReduce::count_included() +------------------------------------------------------------------------- */ + +bigint ComputeReduceKokkos::count_included_kokkos() +{ + // the surf ownership decomposition and masks are host-only + + if (flavor[0] == SURF) { + sync_host_for_fallback(0); + return ComputeReduce::count_included(); + } + + bigint ncount = 0; + bigint ncountall = 0; + + if (!subsetID) { + if (flavor[0] == PARTICLE) ncount = particle->nlocal; + else ncount = grid->nlocal; + + } else { + build_include(0); + + // an integer count is order independent, so no SPARTA_KOKKOS_EXACT + // special case is needed here + + bigint nsum = 0; + if (nelements) { + auto l_include = d_include; + Kokkos::parallel_reduce(Kokkos::RangePolicy(0,nelements), + KOKKOS_LAMBDA(const int i, bigint &lsum) { + lsum += l_include(i); + },nsum); + } + ncount = nsum; + } + + MPI_Allreduce(&ncount,&ncountall,1,MPI_SPARTA_BIGINT,MPI_SUM,world); + + return ncountall; +} + +/* ---------------------------------------------------------------------- */ + +void ComputeReduceKokkos::grow_scratch(int n) +{ + if (n <= maxelements) return; + maxelements = n; + MemKK::realloc_kokkos(d_values,"reduce/kk:values",maxelements); + MemKK::realloc_kokkos(d_include,"reduce/kk:include",maxelements); +} diff --git a/src/KOKKOS/compute_reduce_kokkos.h b/src/KOKKOS/compute_reduce_kokkos.h new file mode 100644 index 000000000..7c223bba4 --- /dev/null +++ b/src/KOKKOS/compute_reduce_kokkos.h @@ -0,0 +1,115 @@ +/* ---------------------------------------------------------------------- + SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer + http://sparta.github.io + Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov + Sandia National Laboratories + + Copyright (2014) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level SPARTA directory. +------------------------------------------------------------------------- */ + +#ifdef COMPUTE_CLASS + +ComputeStyle(reduce/kk,ComputeReduceKokkos) + +#else + +#ifndef SPARTA_COMPUTE_REDUCE_KOKKOS_H +#define SPARTA_COMPUTE_REDUCE_KOKKOS_H + +#include "compute_reduce.h" +#include "kokkos_base.h" +#include "kokkos_type.h" + +namespace SPARTA_NS { + +class ComputeReduceKokkos : public ComputeReduce, public KokkosBase { + public: + ComputeReduceKokkos(class SPARTA *, int, char **); + ~ComputeReduceKokkos(); + void init(); + double compute_scalar(); + void compute_vector(); + + // NOTE: the KokkosBase per-grid/per-particle views are intentionally left + // unused by this style. compute reduce only produces a global scalar or + // global vector, so there is no device-resident output for a downstream + // Kokkos style to read. KokkosBase is inherited so that this compute + // looks like every other /kk compute to a dynamic_cast + + // --------------------------------------------------------------------- + // DANGER, DO NOT "SIMPLIFY" THE KERNELS BELOW INTO TAGGED FUNCTORS + // + // ComputeReduce::~ComputeReduce() has NO "if (copymode) return;" guard + // (verified against src/compute_reduce.cpp: it unconditionally deletes + // which/argindex/flavor/ids/value2index/replace/vector/onevec/indices/ + // owner/subsetID and destroys varparticle/vargrid/varsurf/smasks/ + // areasurf). Handing *this to Kokkos::parallel_for/parallel_reduce + // copies the object; when that copy is destroyed the base destructor + // runs again and double-frees all of the above. Setting copymode=1 + // does NOT help, because the base destructor never tests it. + // + // Therefore every kernel in the .cpp is a KOKKOS_LAMBDA over LOCAL copies + // of the views it touches. No lambda may name a data member: KOKKOS_ + // LAMBDA expands to [=] and naming a member would capture "this", which + // on a CUDA/HIP backend is a host pointer dereferenced in device code. + // Copy what you need into a local first. + // + // These helpers are public on purpose. nvcc's extended-lambda rules + // forbid defining a __device__ lambda inside a member function that has + // private or protected access in its class, and each of them defines + // one. Do not move them into the private section. + // --------------------------------------------------------------------- + + double compute_one_kokkos(int, int); + int setup_values(int); + void build_include(int); + void gather_float(DAT::t_float_1d_strided); + void gather_int_vec(DAT::t_int_1d); + void gather_int_array(DAT::t_int_2d_lr, int); + double reduce_values(); + bigint count_included_kokkos(); + void grow_scratch(int); + void sync_host_for_fallback(int); + + private: + + // gathered per-element values for the input currently being reduced, plus + // a 0/1 flag per element for membership in the subset (particle mixture + // or grid group). both are indexed 0 <= i < nelements, where nelements + // is particle->nlocal for PARTICLE inputs and grid->nlocal for GRID ones + + DAT::t_float_1d d_values; + DAT::t_int_1d d_include; + int nelements,maxelements; + + // device copy of ComputeReduce::s2g, refreshed from the host pointer at + // every use so it can never disagree with, or be shorter than, the + // mixture map the non-Kokkos path would have read + + DAT::tdual_int_1d k_s2g; +}; + +} + +#endif +#endif + +/* ERROR/WARNING messages: + +E: Illegal ... command + +Self-explanatory. Check the input script syntax and compare to the +documentation for the command. You can use -echo screen as a +command-line option when running SPARTA to see the offending line. + +E: Fix used in compute reduce not computed at compatible time + +Fixes generate their values on specific timesteps. Compute reduce is +requesting a value on a non-allowed timestep. + +*/ diff --git a/src/KOKKOS/compute_surf_collision_tally_kokkos.cpp b/src/KOKKOS/compute_surf_collision_tally_kokkos.cpp new file mode 100644 index 000000000..12ff02ec6 --- /dev/null +++ b/src/KOKKOS/compute_surf_collision_tally_kokkos.cpp @@ -0,0 +1,191 @@ +/* ---------------------------------------------------------------------- + SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer + http://sparta.github.io + Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov + Sandia National Laboratories + + Copyright (2014) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level SPARTA directory. +------------------------------------------------------------------------- */ + +#include "compute_surf_collision_tally_kokkos.h" +#include +#include "particle_kokkos.h" +#include "surf_kokkos.h" +#include "update.h" +#include "domain.h" +#include "mixture.h" +#include "memory_kokkos.h" +#include "sparta_masks.h" +#include "error.h" + +using namespace SPARTA_NS; + +#define DELTA 4096 + +/* ---------------------------------------------------------------------- */ + +ComputeSurfCollisionTallyKokkos::ComputeSurfCollisionTallyKokkos(SPARTA *sparta, + int narg, char **arg) : + ComputeSurfCollisionTally(sparta, narg, arg) +{ + kokkos_flag = 1; + ntally_mark = 0; + + d_ntally = DAT::t_int_scalar("surf/collision/tally/kk:ntally"); + h_ntally = HAT::t_int_scalar("surf/collision/tally/kk:ntally_mirror"); + + // flatten the value list once; it never changes after construction + + DAT::tdual_int_1d k_which("surf/collision/tally/kk:which",nvalue); + for (int m = 0; m < nvalue; m++) k_which.view_host()[m] = which[m]; + k_which.modify_host(); + k_which.sync_device(); + d_which = k_which.view_device(); + + maxtally = DELTA; + maxtally_host = 0; + MemKK::realloc_kokkos(k_array_tally,"surf/collision/tally/kk:array_tally", + maxtally,nvalue); + d_array_tally = k_array_tally.view_device(); +} + +/* ---------------------------------------------------------------------- */ + +ComputeSurfCollisionTallyKokkos::ComputeSurfCollisionTallyKokkos(SPARTA *sparta) : + ComputeSurfCollisionTally(sparta) +{ + copy = 1; + ntally_mark = 0; +} + +/* ---------------------------------------------------------------------- */ + +ComputeSurfCollisionTallyKokkos::~ComputeSurfCollisionTallyKokkos() +{ + if (copy || copymode) return; + + // the host base class frees array_tally, which this class never allocated + + memory->destroy(array_tally); + array_tally = NULL; + maxtally = 0; +} + +/* ---------------------------------------------------------------------- */ + +void ComputeSurfCollisionTallyKokkos::clear() +{ + ntally = 0; +} + +/* ---------------------------------------------------------------------- + called by UpdateKokkos before the move kernel +------------------------------------------------------------------------- */ + +void ComputeSurfCollisionTallyKokkos::pre_surf_tally() +{ + SurfKokkos* surf_kk = (SurfKokkos*) surf; + surf_kk->sync(Device,LINE_MASK|TRI_MASK); + d_lines = surf_kk->k_lines.view_device(); + d_tris = surf_kk->k_tris.view_device(); + + ParticleKokkos* particle_kk = (ParticleKokkos*) particle; + particle_kk->sync(Device,SPECIES_MASK); + d_s2g = particle_kk->k_species2group.view_device(); + + dim = domain->dimension; + dt = update->dt; + + Kokkos::deep_copy(d_ntally,0); +} + +/* ---------------------------------------------------------------------- + called by UpdateKokkos after the move kernel + bring the row count and the rows themselves to the host, where + dump tally and Compute::tallyinfo() read them +------------------------------------------------------------------------- */ + +void ComputeSurfCollisionTallyKokkos::post_surf_tally() +{ + Kokkos::deep_copy(h_ntally,d_ntally); + ntally = h_ntally(); + + // an overflowed attempt is discarded and repeated by UpdateKokkos, so do + // not publish its partial rows. the device counter kept climbing past + // the end of the buffer, so ntally is not a row count here -- leaving it + // in the host base class would make dump tally read that many rows out + // of an array that never held them + + if (ntally > (int) d_array_tally.extent(0)) { ntally = 0; return; } + + // the host base class hands out array_tally, row-indexed to ntally + // (compute_surf_collision_tally.cpp:172). Publish only the rows this + // step produced: k_array_tally is sized to maxtally, a high-water mark, + // so sync_host() moved the whole buffer and the scalar loop then touched + // every element of it again. The host half of the dual view has no + // other reader, so copy the device rows straight into array_tally -- + // d_array_tally is LayoutRight and Memory::create hands back one + // contiguous row-major block with the same nvalue row stride, so an + // unmanaged wrap of the leading ntally rows lines up exactly. Keep the + // allocation on its own high-water mark rather than free/malloc-ing it + // every step. + + if (ntally) { + static_assert(std::is_same::value, + "array_tally is double**, so F_FLOAT must be double " + "for the unmanaged wrap below to alias it"); + if (ntally > maxtally_host) { + memory->destroy(array_tally); + maxtally_host = MAX(ntally,maxtally); + memory->create(array_tally,maxtally_host,nvalue, + "surf/collision/tally/kk:array_tally_host"); + } + Kokkos::View > + h_rows(array_tally[0],ntally,nvalue); + Kokkos::deep_copy(h_rows, + Kokkos::subview(d_array_tally, + Kokkos::make_pair(0,ntally), + Kokkos::ALL())); + } +} + +/* ---------------------------------------------------------------------- + grow the device row buffer to hold at least N rows + called by UpdateKokkos when an attempt overflowed, before repeating it +------------------------------------------------------------------------- */ + +void ComputeSurfCollisionTallyKokkos::grow_tally_kokkos(int n) +{ + if (n <= (int) d_array_tally.extent(0)) return; + + // grow past what the failed attempt asked for, so a step whose collision + // count is still climbing does not repeat the move again and again + + maxtally = MAX(n + DELTA, (int)(1.5*n)); + + // resize, not realloc: MemKK::realloc_kokkos drops the old allocation and + // allocates WithoutInitializing, so every existing row becomes garbage. + // The rows below ntally_mark were written by earlier migration iterations + // of this same step and must survive -- rewind_ntally() only takes the + // count back to the mark, and the re-run appends from there, so a + // discarded prefix would be published as uninitialized tally output. + // DualView::resize preserves contents, as memoryKK->grow_kokkos does. + // + // No deck reaches that combination today: maxtally is a high-water mark + // that persists across steps, so by the time a step has more than one + // tallying migration iteration the buffer already fits the whole step and + // never overflows again. Probing in.surf.collision.tally at 4 ranks with + // DELTA forced to 8 and then to no headroom at all gave 13 grows, every + // one with ntally_mark == 0, against 40 nonzero marks elsewhere in the + // same run. This is a latent hazard closed on principle, not a + // reproduced failure. + + k_array_tally.resize(maxtally,nvalue); + d_array_tally = k_array_tally.view_device(); +} diff --git a/src/KOKKOS/compute_surf_collision_tally_kokkos.h b/src/KOKKOS/compute_surf_collision_tally_kokkos.h new file mode 100644 index 000000000..c1ba53c2b --- /dev/null +++ b/src/KOKKOS/compute_surf_collision_tally_kokkos.h @@ -0,0 +1,147 @@ +/* ---------------------------------------------------------------------- + SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer + http://sparta.github.io + Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov + Sandia National Laboratories + + Copyright (2014) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level SPARTA directory. +------------------------------------------------------------------------- */ + +#ifdef COMPUTE_CLASS + +ComputeStyle(surf/collision/tally/kk,ComputeSurfCollisionTallyKokkos) + +#else + +#ifndef SPARTA_COMPUTE_SURF_COLLISION_TALLY_KOKKOS_H +#define SPARTA_COMPUTE_SURF_COLLISION_TALLY_KOKKOS_H + +#include "compute_surf_collision_tally.h" +#include "kokkos_base.h" +#include "kokkos_type.h" + +namespace SPARTA_NS { + +// unlike the per-grid and per-surf tally computes, this one appends one row +// per collision event, so the row count is not known before the move +// kernel runs. rows are claimed with an atomic counter; if a claim lands +// past the end of the buffer the row is dropped and d_tally_overflow is +// raised, which makes UpdateKokkos grow every such compute and repeat the +// move. truncating instead would silently corrupt dump tally output. + +class ComputeSurfCollisionTallyKokkos : public ComputeSurfCollisionTally, public KokkosBase { + public: + ComputeSurfCollisionTallyKokkos(class SPARTA *, int, char **); + ComputeSurfCollisionTallyKokkos(class SPARTA *); + ~ComputeSurfCollisionTallyKokkos() override; + + void clear() override; + + // called by UpdateKokkos around the move kernel + + void pre_surf_tally(); + void post_surf_tally(); + void grow_tally_kokkos(int); + + // grow to what the overflowed attempt actually needed; the device counter + // kept counting past the end of the buffer, so it is that number + + void grow_after_overflow() + { + Kokkos::deep_copy(h_ntally,d_ntally); + grow_tally_kokkos(h_ntally()); + } + + // every attempt of the retry loop re-runs the whole pass, so the rows the + // aborted attempt appended have to be taken back before the next one. + // Not by zeroing: the move kernel runs once per migration iteration and + // the tally accumulates across all of them, so an attempt rewinds to + // where its own pass started, which mark_ntally() records. + + void mark_ntally() + { + Kokkos::deep_copy(h_ntally,d_ntally); + ntally_mark = h_ntally(); + } + + void rewind_ntally() { Kokkos::deep_copy(d_ntally,ntally_mark); } + + DAT::t_int_scalar d_overflow; // set by UpdateKokkos each step + + KOKKOS_INLINE_FUNCTION + void surf_tally_kk(double dtremain, int isurf, int icell, int reaction, + Particle::OnePart *iorig, + Particle::OnePart *ip, Particle::OnePart *jp) const + { + // this compute tallies only collisions that induce no reaction; + // reactions belong to compute surf/reaction/tally + + if (reaction) return; + + if (dim == 2) { + if (!(d_lines(isurf).mask & groupbit)) return; + } else { + if (!(d_tris(isurf).mask & groupbit)) return; + } + + const int origspecies = iorig->ispecies; + if (d_s2g(imix,origspecies) < 0) return; + + const int itally = Kokkos::atomic_fetch_add(&d_ntally(),1); + if (itally >= (int) d_array_tally.extent(0)) { + d_overflow() = 1; + return; + } + + for (int m = 0; m < nvalue; m++) { + switch (d_which[m]) { + case IDSURF: + if (dim == 2) d_array_tally(itally,m) = ubuf(d_lines(isurf).id).d; + else d_array_tally(itally,m) = ubuf(d_tris(isurf).id).d; + break; + case ID: d_array_tally(itally,m) = ubuf(ip->id).d; break; + case TYPE: d_array_tally(itally,m) = ubuf(ip->ispecies+1).d; break; + case XC: d_array_tally(itally,m) = iorig->x[0]; break; + case YC: d_array_tally(itally,m) = iorig->x[1]; break; + case ZC: d_array_tally(itally,m) = iorig->x[2]; break; + case TIME: d_array_tally(itally,m) = dt - dtremain; break; + case VXPRE: d_array_tally(itally,m) = iorig->v[0]; break; + case VYPRE: d_array_tally(itally,m) = iorig->v[1]; break; + case VZPRE: d_array_tally(itally,m) = iorig->v[2]; break; + case VXPOST: d_array_tally(itally,m) = ip->v[0]; break; + case VYPOST: d_array_tally(itally,m) = ip->v[1]; break; + case VZPOST: d_array_tally(itally,m) = ip->v[2]; break; + } + } + } + + private: + // must match the enum in compute_surf_collision_tally.cpp + + enum{IDSURF,ID,TYPE,TIME,XC,YC,ZC,VXPRE,VYPRE,VZPRE,VXPOST,VYPOST,VZPOST}; + + int maxtally_host; // rows array_tally is allocated for + DAT::tdual_float_2d_lr k_array_tally; + DAT::t_float_2d_lr d_array_tally; + DAT::t_int_scalar d_ntally; + int ntally_mark; + HAT::t_int_scalar h_ntally; + + DAT::t_int_1d d_which; + DAT::t_int_2d d_s2g; + + t_line_1d d_lines; + t_tri_1d d_tris; + + double dt; +}; + +} + +#endif +#endif diff --git a/src/KOKKOS/compute_surf_kokkos.cpp b/src/KOKKOS/compute_surf_kokkos.cpp index f2ecb106b..0929c0f75 100644 --- a/src/KOKKOS/compute_surf_kokkos.cpp +++ b/src/KOKKOS/compute_surf_kokkos.cpp @@ -34,9 +34,11 @@ using namespace SPARTA_NS; /* ---------------------------------------------------------------------- */ ComputeSurfKokkos::ComputeSurfKokkos(SPARTA *sparta, int narg, char **arg) : - ComputeSurf(sparta, narg, arg), - sr_kk_global_copy{VAL_2(KKCopy(sparta))}, - sr_kk_prob_copy{VAL_2(KKCopy(sparta))} + ComputeSurf(sparta, narg, arg) +#ifdef SPARTA_KOKKOS_FIXED_LISTS + , sr_kk_global_copy{VAL_2(KKCopy(sparta))} + , sr_kk_prob_copy{VAL_2(KKCopy(sparta))} +#endif { kokkos_flag = 1; compressed = 0; @@ -44,9 +46,11 @@ ComputeSurfKokkos::ComputeSurfKokkos(SPARTA *sparta, int narg, char **arg) : } ComputeSurfKokkos::ComputeSurfKokkos(SPARTA *sparta) : - ComputeSurf(sparta), - sr_kk_global_copy{VAL_2(KKCopy(sparta))}, - sr_kk_prob_copy{VAL_2(KKCopy(sparta))} + ComputeSurf(sparta) +#ifdef SPARTA_KOKKOS_FIXED_LISTS + , sr_kk_global_copy{VAL_2(KKCopy(sparta))} + , sr_kk_prob_copy{VAL_2(KKCopy(sparta))} +#endif { copy = 1; compressed = 0; @@ -108,6 +112,15 @@ void ComputeSurfKokkos::init_normflux() void ComputeSurfKokkos::clear() { + // tallyinfo() compresses the tally list in place on the host and claims the + // host for it. A new cycle starts here: the device copy below is about to be + // zeroed and refilled by the tally kernels, so the two are deliberately + // parted and neither owes the other a copy. Without this the modify_device() + // in post_surf_tally() would meet the outstanding host claim and abort. + + k_tally2surf.clear_sync_state(); + k_array_surf_tally.clear_sync_state(); + // reset all set surf2tally values to -1 // called by Update at beginning of timesteps surf tallying is done @@ -142,8 +155,21 @@ void ComputeSurfKokkos::pre_surf_tally() else ndup_array_surf_tally = Kokkos::Experimental::create_scatter_view(d_array_surf_tally); +#ifdef SPARTA_KOKKOS_FIXED_LISTS if (surf->nsr > KOKKOS_MAX_TOT_SURF_REACT) error->all(FLERR,"Kokkos currently supports a limited number of surface reaction methods"); +#else + + // the buffers must be sized before anything is blitted into them. surf->nsr + // bounds the count of every individual style, so sizing both of them to it + // needs no counting pass, and the loop below still runs pre_react() in + // surf react list order + + sr_idx_resize(k_sr_type_list,d_sr_type_list,surf->nsr); + sr_idx_resize(k_sr_map,d_sr_map,surf->nsr); + sr_buf_resize(k_sr_global,d_sr_global,surf->nsr); + sr_buf_resize(k_sr_prob,d_sr_prob,surf->nsr); +#endif if (surf->nsr > 0) { int nglob,nprob; @@ -152,25 +178,44 @@ void ComputeSurfKokkos::pre_surf_tally() if (!surf->sr[n]->kokkosable) error->all(FLERR,"Must use Kokkos-enabled surface reaction method with Kokkos"); if (strcmp(surf->sr[n]->style,"global") == 0) { +#ifdef SPARTA_KOKKOS_FIXED_LISTS if (nglob >= KOKKOS_MAX_SURF_REACT_PER_TYPE) error->all(FLERR,"Kokkos currently supports two instances of each surface reaction method"); sr_kk_global_copy[nglob].copy((SurfReactGlobalKokkos*)(surf->sr[n])); - sr_kk_global_copy[nglob].obj.pre_react(); - sr_type_list[n] = 0; - sr_map[n] = nglob; +#else + sr_buf_blit(k_sr_global,nglob,(SurfReactGlobalKokkos*)(surf->sr[n])); +#endif + KK_SR_H_GLOBAL(nglob).pre_react(); + KK_SR_H_TYPE(n) = 0; + KK_SR_H_MAP(n) = nglob; nglob++; } else if (strcmp(surf->sr[n]->style,"prob") == 0) { +#ifdef SPARTA_KOKKOS_FIXED_LISTS if (nprob >= KOKKOS_MAX_SURF_REACT_PER_TYPE) error->all(FLERR,"Kokkos currently supports two instances of each surface reaction method"); sr_kk_prob_copy[nprob].copy((SurfReactProbKokkos*)(surf->sr[n])); - sr_kk_prob_copy[nprob].obj.pre_react(); - sr_type_list[n] = 1; - sr_map[n] = nprob; +#else + sr_buf_blit(k_sr_prob,nprob,(SurfReactProbKokkos*)(surf->sr[n])); +#endif + KK_SR_H_PROB(nprob).pre_react(); + KK_SR_H_TYPE(n) = 1; + KK_SR_H_MAP(n) = nprob; nprob++; } else { error->all(FLERR,"Unknown Kokkos surface reaction method"); } } + +#ifndef SPARTA_KOKKOS_FIXED_LISTS + + // the models were blitted into the host image of the buffers and their + // pre_react() ran there; push the result to the device + + sr_buf_sync(k_sr_global,d_sr_global); + sr_buf_sync(k_sr_prob,d_sr_prob); + sr_idx_sync(k_sr_type_list,d_sr_type_list); + sr_idx_sync(k_sr_map,d_sr_map); +#endif } } @@ -192,8 +237,8 @@ void ComputeSurfKokkos::post_surf_tally() // orphaned and its allocation never freed for (int n = 0; n < surf->nsr; n++) { - if (sr_type_list[n] == 0) sr_kk_global_copy[sr_map[n]].obj.post_react(); - else sr_kk_prob_copy[sr_map[n]].obj.post_react(); + if (KK_SR_H_TYPE(n) == 0) KK_SR_H_GLOBAL(KK_SR_H_MAP(n)).post_react(); + else KK_SR_H_PROB(KK_SR_H_MAP(n)).post_react(); } } @@ -239,6 +284,14 @@ int ComputeSurfKokkos::tallyinfo(surfint *&ptr) tally2surf[istart] = tally2surf[iend]; } + // the compression above rewrote both arrays on the host. Claim it: the dense + // list is what every consumer reads, and an unclaimed write is discarded by + // the next sync_host() or by the realloc in init_normflux(), which resizes + // whichever side the counters call newer. + + k_tally2surf.modify_host(); + k_array_surf_tally.modify_host(); + return ntally; } diff --git a/src/KOKKOS/compute_surf_kokkos.h b/src/KOKKOS/compute_surf_kokkos.h index 25021fe44..aac1b40b0 100644 --- a/src/KOKKOS/compute_surf_kokkos.h +++ b/src/KOKKOS/compute_surf_kokkos.h @@ -404,11 +404,11 @@ void surf_tally_kk(double /*dtremain*/, int isurf, int icell, int reaction, break; case ECHEM: if (reaction && !transparent) { - int sr_type = sr_type_list[isr]; - int m = sr_map[isr]; + int sr_type = KK_SR_TYPE(isr); + int m = KK_SR_MAP(isr); double r_coeff = 0.0; if (sr_type == 1) - r_coeff = sr_kk_prob_copy[m].obj.d_coeffs(reaction-1,1); + r_coeff = KK_SR_PROB(m).d_coeffs(reaction-1,1); a_array_surf_tally(itally,k) += weight * r_coeff * fluxscale; } k++; @@ -431,11 +431,11 @@ void surf_tally_kk(double /*dtremain*/, int isurf, int icell, int reaction, etot = 0.5*mvv2e*(ivsqpost + jvsqpost - vsqpre) + weight * (iother + jother - otherpre); if (reaction) { - int sr_type = sr_type_list[isr]; - int m = sr_map[isr]; + int sr_type = KK_SR_TYPE(isr); + int m = KK_SR_MAP(isr); double r_coeff = 0.0; if (sr_type == 1) - r_coeff = sr_kk_prob_copy[m].obj.d_coeffs(reaction-1,1); + r_coeff = KK_SR_PROB(m).d_coeffs(reaction-1,1); etot -= weight * r_coeff; } } @@ -470,10 +470,22 @@ void surf_tally_kk(double /*dtremain*/, int isurf, int icell, int reaction, t_line_1d d_lines; t_tri_1d d_tris; + // the active surf react models this compute may be asked about, partitioned + // by style. Two representations, selected by SPARTA_KOKKOS_FIXED_LISTS + // (see kokkos_type.h); the device sites in surf_tally_kk() above are + // written once, against the KK_SR_* accessors defined there. + +#ifdef SPARTA_KOKKOS_FIXED_LISTS int sr_type_list[KOKKOS_MAX_TOT_SURF_REACT]; int sr_map[KOKKOS_MAX_TOT_SURF_REACT]; KKCopy sr_kk_global_copy[KOKKOS_MAX_SURF_REACT_PER_TYPE]; KKCopy sr_kk_prob_copy[KOKKOS_MAX_SURF_REACT_PER_TYPE]; +#else + DAT::tdual_int_1d k_sr_type_list,k_sr_map; + DAT::t_int_1d d_sr_type_list,d_sr_map; + DAT::tdual_char_1d k_sr_global,k_sr_prob; + DAT::t_char_1d d_sr_global,d_sr_prob; +#endif void grow_tally(); }; diff --git a/src/KOKKOS/compute_surf_reaction_tally_kokkos.cpp b/src/KOKKOS/compute_surf_reaction_tally_kokkos.cpp new file mode 100644 index 000000000..aeaa004c5 --- /dev/null +++ b/src/KOKKOS/compute_surf_reaction_tally_kokkos.cpp @@ -0,0 +1,191 @@ +/* ---------------------------------------------------------------------- + SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer + http://sparta.github.io + Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov + Sandia National Laboratories + + Copyright (2014) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level SPARTA directory. +------------------------------------------------------------------------- */ + +#include "compute_surf_reaction_tally_kokkos.h" +#include +#include "particle_kokkos.h" +#include "surf_kokkos.h" +#include "update.h" +#include "domain.h" +#include "mixture.h" +#include "memory_kokkos.h" +#include "sparta_masks.h" +#include "error.h" + +using namespace SPARTA_NS; + +#define DELTA 4096 + +/* ---------------------------------------------------------------------- */ + +ComputeSurfReactionTallyKokkos::ComputeSurfReactionTallyKokkos(SPARTA *sparta, + int narg, char **arg) : + ComputeSurfReactionTally(sparta, narg, arg) +{ + kokkos_flag = 1; + ntally_mark = 0; + + d_ntally = DAT::t_int_scalar("surf/reaction/tally/kk:ntally"); + h_ntally = HAT::t_int_scalar("surf/reaction/tally/kk:ntally_mirror"); + + // flatten the value list once; it never changes after construction + + DAT::tdual_int_1d k_which("surf/reaction/tally/kk:which",nvalue); + for (int m = 0; m < nvalue; m++) k_which.view_host()[m] = which[m]; + k_which.modify_host(); + k_which.sync_device(); + d_which = k_which.view_device(); + + maxtally = DELTA; + maxtally_host = 0; + MemKK::realloc_kokkos(k_array_tally,"surf/reaction/tally/kk:array_tally", + maxtally,nvalue); + d_array_tally = k_array_tally.view_device(); +} + +/* ---------------------------------------------------------------------- */ + +ComputeSurfReactionTallyKokkos::ComputeSurfReactionTallyKokkos(SPARTA *sparta) : + ComputeSurfReactionTally(sparta) +{ + copy = 1; + ntally_mark = 0; +} + +/* ---------------------------------------------------------------------- */ + +ComputeSurfReactionTallyKokkos::~ComputeSurfReactionTallyKokkos() +{ + if (copy || copymode) return; + + // the host base class frees array_tally, which this class never allocated + + memory->destroy(array_tally); + array_tally = NULL; + maxtally = 0; +} + +/* ---------------------------------------------------------------------- */ + +void ComputeSurfReactionTallyKokkos::clear() +{ + ntally = 0; +} + +/* ---------------------------------------------------------------------- + called by UpdateKokkos before the move kernel +------------------------------------------------------------------------- */ + +void ComputeSurfReactionTallyKokkos::pre_surf_tally() +{ + SurfKokkos* surf_kk = (SurfKokkos*) surf; + surf_kk->sync(Device,LINE_MASK|TRI_MASK); + d_lines = surf_kk->k_lines.view_device(); + d_tris = surf_kk->k_tris.view_device(); + + ParticleKokkos* particle_kk = (ParticleKokkos*) particle; + particle_kk->sync(Device,SPECIES_MASK); + d_s2g = particle_kk->k_species2group.view_device(); + + dim = domain->dimension; + dt = update->dt; + + Kokkos::deep_copy(d_ntally,0); +} + +/* ---------------------------------------------------------------------- + called by UpdateKokkos after the move kernel + bring the row count and the rows themselves to the host, where + dump tally and Compute::tallyinfo() read them +------------------------------------------------------------------------- */ + +void ComputeSurfReactionTallyKokkos::post_surf_tally() +{ + Kokkos::deep_copy(h_ntally,d_ntally); + ntally = h_ntally(); + + // an overflowed attempt is discarded and repeated by UpdateKokkos, so do + // not publish its partial rows. the device counter kept climbing past + // the end of the buffer, so ntally is not a row count here -- leaving it + // in the host base class would make dump tally read that many rows out + // of an array that never held them + + if (ntally > (int) d_array_tally.extent(0)) { ntally = 0; return; } + + // the host base class hands out array_tally, row-indexed to ntally + // (compute_surf_collision_tally.cpp:172). Publish only the rows this + // step produced: k_array_tally is sized to maxtally, a high-water mark, + // so sync_host() moved the whole buffer and the scalar loop then touched + // every element of it again. The host half of the dual view has no + // other reader, so copy the device rows straight into array_tally -- + // d_array_tally is LayoutRight and Memory::create hands back one + // contiguous row-major block with the same nvalue row stride, so an + // unmanaged wrap of the leading ntally rows lines up exactly. Keep the + // allocation on its own high-water mark rather than free/malloc-ing it + // every step. + + if (ntally) { + static_assert(std::is_same::value, + "array_tally is double**, so F_FLOAT must be double " + "for the unmanaged wrap below to alias it"); + if (ntally > maxtally_host) { + memory->destroy(array_tally); + maxtally_host = MAX(ntally,maxtally); + memory->create(array_tally,maxtally_host,nvalue, + "surf/reaction/tally/kk:array_tally_host"); + } + Kokkos::View > + h_rows(array_tally[0],ntally,nvalue); + Kokkos::deep_copy(h_rows, + Kokkos::subview(d_array_tally, + Kokkos::make_pair(0,ntally), + Kokkos::ALL())); + } +} + +/* ---------------------------------------------------------------------- + grow the device row buffer to hold at least N rows + called by UpdateKokkos when an attempt overflowed, before repeating it +------------------------------------------------------------------------- */ + +void ComputeSurfReactionTallyKokkos::grow_tally_kokkos(int n) +{ + if (n <= (int) d_array_tally.extent(0)) return; + + // grow past what the failed attempt asked for, so a step whose collision + // count is still climbing does not repeat the move again and again + + maxtally = MAX(n + DELTA, (int)(1.5*n)); + + // resize, not realloc: MemKK::realloc_kokkos drops the old allocation and + // allocates WithoutInitializing, so every existing row becomes garbage. + // The rows below ntally_mark were written by earlier migration iterations + // of this same step and must survive -- rewind_ntally() only takes the + // count back to the mark, and the re-run appends from there, so a + // discarded prefix would be published as uninitialized tally output. + // DualView::resize preserves contents, as memoryKK->grow_kokkos does. + // + // No deck reaches that combination today: maxtally is a high-water mark + // that persists across steps, so by the time a step has more than one + // tallying migration iteration the buffer already fits the whole step and + // never overflows again. Probing in.surf.collision.tally at 4 ranks with + // DELTA forced to 8 and then to no headroom at all gave 13 grows, every + // one with ntally_mark == 0, against 40 nonzero marks elsewhere in the + // same run. This is a latent hazard closed on principle, not a + // reproduced failure. + + k_array_tally.resize(maxtally,nvalue); + d_array_tally = k_array_tally.view_device(); +} diff --git a/src/KOKKOS/compute_surf_reaction_tally_kokkos.h b/src/KOKKOS/compute_surf_reaction_tally_kokkos.h new file mode 100644 index 000000000..f782356cb --- /dev/null +++ b/src/KOKKOS/compute_surf_reaction_tally_kokkos.h @@ -0,0 +1,156 @@ +/* ---------------------------------------------------------------------- + SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer + http://sparta.github.io + Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov + Sandia National Laboratories + + Copyright (2014) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level SPARTA directory. +------------------------------------------------------------------------- */ + +#ifdef COMPUTE_CLASS + +ComputeStyle(surf/reaction/tally/kk,ComputeSurfReactionTallyKokkos) + +#else + +#ifndef SPARTA_COMPUTE_SURF_REACTION_TALLY_KOKKOS_H +#define SPARTA_COMPUTE_SURF_REACTION_TALLY_KOKKOS_H + +#include "compute_surf_reaction_tally.h" +#include "kokkos_base.h" +#include "kokkos_type.h" + +namespace SPARTA_NS { + +// unlike the per-grid and per-surf tally computes, this one appends one row +// per collision event, so the row count is not known before the move +// kernel runs. rows are claimed with an atomic counter; if a claim lands +// past the end of the buffer the row is dropped and d_tally_overflow is +// raised, which makes UpdateKokkos grow every such compute and repeat the +// move. truncating instead would silently corrupt dump tally output. + +class ComputeSurfReactionTallyKokkos : public ComputeSurfReactionTally, public KokkosBase { + public: + ComputeSurfReactionTallyKokkos(class SPARTA *, int, char **); + ComputeSurfReactionTallyKokkos(class SPARTA *); + ~ComputeSurfReactionTallyKokkos() override; + + void clear() override; + + // called by UpdateKokkos around the move kernel + + void pre_surf_tally(); + void post_surf_tally(); + void grow_tally_kokkos(int); + + // grow to what the overflowed attempt actually needed; the device counter + // kept counting past the end of the buffer, so it is that number + + void grow_after_overflow() + { + Kokkos::deep_copy(h_ntally,d_ntally); + grow_tally_kokkos(h_ntally()); + } + + // every attempt of the retry loop re-runs the whole pass, so the rows the + // aborted attempt appended have to be taken back before the next one. + // Not by zeroing: the move kernel runs once per migration iteration and + // the tally accumulates across all of them, so an attempt rewinds to + // where its own pass started, which mark_ntally() records. + + void mark_ntally() + { + Kokkos::deep_copy(h_ntally,d_ntally); + ntally_mark = h_ntally(); + } + + void rewind_ntally() { Kokkos::deep_copy(d_ntally,ntally_mark); } + + DAT::t_int_scalar d_overflow; // set by UpdateKokkos each step + + KOKKOS_INLINE_FUNCTION + void surf_tally_kk(double dtremain, int isurf, int icell, int reaction, + Particle::OnePart *iorig, + Particle::OnePart *ip, Particle::OnePart *jp) const + { + // this compute tallies only collisions that induce a reaction; + // plain collisions belong to compute surf/collision/tally + + if (!reaction) return; + + if (dim == 2) { + if (!(d_lines(isurf).mask & groupbit)) return; + } else { + if (!(d_tris(isurf).mask & groupbit)) return; + } + + const int origspecies = iorig->ispecies; + if (d_s2g(imix,origspecies) < 0) return; + + const int itally = Kokkos::atomic_fetch_add(&d_ntally(),1); + if (itally >= (int) d_array_tally.extent(0)) { + d_overflow() = 1; + return; + } + + for (int m = 0; m < nvalue; m++) { + switch (d_which[m]) { + case REACTION: d_array_tally(itally,m) = ubuf(reaction).d; break; + case IDSURF: + if (dim == 2) d_array_tally(itally,m) = ubuf(d_lines(isurf).id).d; + else d_array_tally(itally,m) = ubuf(d_tris(isurf).id).d; + break; + case IDPRE: d_array_tally(itally,m) = ubuf(iorig->id).d; break; + case ID1POST: d_array_tally(itally,m) = ip ? ubuf(ip->id).d : ubuf(0).d; break; + case ID2POST: d_array_tally(itally,m) = jp ? ubuf(jp->id).d : ubuf(0).d; break; + case TYPEPRE: d_array_tally(itally,m) = ubuf(iorig->ispecies+1).d; break; + case TYPE1POST: d_array_tally(itally,m) = ip ? ubuf(ip->ispecies+1).d : ubuf(0).d; break; + case TYPE2POST: d_array_tally(itally,m) = jp ? ubuf(jp->ispecies+1).d : ubuf(0).d; break; + case TIME: d_array_tally(itally,m) = dt - dtremain; break; + case XC: d_array_tally(itally,m) = iorig->x[0]; break; + case YC: d_array_tally(itally,m) = iorig->x[1]; break; + case ZC: d_array_tally(itally,m) = iorig->x[2]; break; + case VXPRE: d_array_tally(itally,m) = iorig->v[0]; break; + case VYPRE: d_array_tally(itally,m) = iorig->v[1]; break; + case VZPRE: d_array_tally(itally,m) = iorig->v[2]; break; + case VX1POST: d_array_tally(itally,m) = ip ? ip->v[0] : 0.0; break; + case VY1POST: d_array_tally(itally,m) = ip ? ip->v[1] : 0.0; break; + case VZ1POST: d_array_tally(itally,m) = ip ? ip->v[2] : 0.0; break; + case VX2POST: d_array_tally(itally,m) = jp ? jp->v[0] : 0.0; break; + case VY2POST: d_array_tally(itally,m) = jp ? jp->v[1] : 0.0; break; + case VZ2POST: d_array_tally(itally,m) = jp ? jp->v[2] : 0.0; break; + } + } + } + + private: + // must match the enum in compute_surf_reaction_tally.cpp + + enum{REACTION,IDSURF,IDPRE,ID1POST,ID2POST,TYPEPRE,TYPE1POST,TYPE2POST,TIME, + XC,YC,ZC,VXPRE,VYPRE,VZPRE,VX1POST,VY1POST,VZ1POST,VX2POST,VY2POST,VZ2POST}; + + int maxtally_host; // rows array_tally is allocated for + DAT::tdual_float_2d_lr k_array_tally; + DAT::t_float_2d_lr d_array_tally; + DAT::t_int_scalar d_ntally; + int ntally_mark; + HAT::t_int_scalar h_ntally; + + DAT::t_int_1d d_which; + DAT::t_int_2d d_s2g; + + t_line_1d d_lines; + t_tri_1d d_tris; + + double dt; +}; + +} + +#endif +#endif diff --git a/src/KOKKOS/fft2d_kokkos.cpp b/src/KOKKOS/fft2d_kokkos.cpp index fd274a5a1..09512785b 100644 --- a/src/KOKKOS/fft2d_kokkos.cpp +++ b/src/KOKKOS/fft2d_kokkos.cpp @@ -66,6 +66,10 @@ FFT2dKokkos::FFT2dKokkos(SPARTA *sparta, MPI_Comm comm, int nfast, i // recursive function calls in KISS FFT and the default per-thread // stack size on GPUs needs to be increased to prevent stack overflows // for reasonably sized FFTs + // NOTE: only CUDA is handled below. HIP has the same recursion and the + // same default-stack problem, and hipDeviceSetLimit(hipLimitStackSize,...) + // is the equivalent call, but it is untested here -- a KISS FFT large + // enough to recurse deeply may still overflow the stack on AMD GPUs. #if defined (KOKKOS_ENABLE_CUDA) size_t stack_size; cudaDeviceGetLimit(&stack_size,cudaLimitStackSize); diff --git a/src/KOKKOS/fft3d_kokkos.cpp b/src/KOKKOS/fft3d_kokkos.cpp index cfa854912..6124c22b0 100644 --- a/src/KOKKOS/fft3d_kokkos.cpp +++ b/src/KOKKOS/fft3d_kokkos.cpp @@ -68,6 +68,10 @@ FFT3dKokkos::FFT3dKokkos(SPARTA *sparta, MPI_Comm comm, int nfast, i // recursive function calls in KISS FFT and the default per-thread // stack size on GPUs needs to be increased to prevent stack overflows // for reasonably sized FFTs + // NOTE: only CUDA is handled below. HIP has the same recursion and the + // same default-stack problem, and hipDeviceSetLimit(hipLimitStackSize,...) + // is the equivalent call, but it is untested here -- a KISS FFT large + // enough to recurse deeply may still overflow the stack on AMD GPUs. #if defined (KOKKOS_ENABLE_CUDA) size_t stack_size; cudaDeviceGetLimit(&stack_size,cudaLimitStackSize); diff --git a/src/KOKKOS/fix_ave_grid_kokkos.cpp b/src/KOKKOS/fix_ave_grid_kokkos.cpp index efd733a05..7d21a9944 100644 --- a/src/KOKKOS/fix_ave_grid_kokkos.cpp +++ b/src/KOKKOS/fix_ave_grid_kokkos.cpp @@ -22,6 +22,7 @@ #include "update.h" #include "modify.h" #include "compute.h" +#include "fix.h" #include "input.h" #include "variable.h" #include "memory_kokkos.h" @@ -215,6 +216,8 @@ void FixAveGridKokkos::end_of_step() modify->clearstep_compute(); + tally_on_host = 0; + for (m = 0; m < nvalues; m++) { n = value2index[m]; j = argindex[m]; @@ -239,6 +242,8 @@ void FixAveGridKokkos::end_of_step() // if compute does not post-process, access its vec/array grid directly // else access uomap columns in its ctally array + tally_to_device(); + if (post_process[m]) { ntally = numap[m]; computeKKBase->query_tally_grid_kokkos(d_ctally); @@ -256,26 +261,86 @@ void FixAveGridKokkos::end_of_step() } // access fix fields, guaranteed to be ready + // two paths, mirroring the host loop in FixAveGrid::end_of_step(): + // fast path: the fix publishes its per-grid output as a device view + // (KokkosBase::d_vector_grid / d_array_grid), so accumulate on device + // with the same kernels used for computes. do NOT gate this on + // fix->kokkos_flag: fix field/grid/kk deliberately clears kokkos_flag + // (its variable evaluation is host-only) yet still publishes a valid + // device array, while fix ave/grid/kk clears it in its PERGRIDSURF + // flavor precisely because the device views are not allocated there. + // the presence of a non-empty device view is the accurate test + // fallback: no usable device view (a non-Kokkos fix such as fix ablate, + // or the PERGRIDSURF flavor of fix ave/grid), so add the host + // per-grid output into the host tally and push it to the device, + // the same round trip the VARIABLE branch below performs } else if (which[m] == FIX) { - error->all(FLERR,"Cannot (yet) use fixes with fix ave/grid/kk"); - //k = umap[m][0]; - //if (j == 0) { - // double *d_fix_vector = modify->fix[n]->vector_grid; // need Kokkos version - // Kokkos::parallel_for(Kokkos::RangePolicy(0,nglocal),*this); - //} else { - // int jm1 = j - 1; - // double **fix_array = modify->fix[n]->array_grid; // need Kokkos version - // Kokkos::parallel_for(Kokkos::RangePolicy(0,nglocal),*this); - //} + Fix *ifix = modify->fix[n]; + KokkosBase *fixKKBase = dynamic_cast(ifix); + if (fixKKBase) fixKKBase->sync_pergrid_device_kokkos(); + k = umap[m][0]; + + if (j == 0) { + int device_ok = fixKKBase && fixKKBase->d_vector_grid.data() && + (int) fixKKBase->d_vector_grid.extent(0) >= nglocal; + + if (device_ok) { + tally_to_device(); + d_fix_vector = fixKKBase->d_vector_grid; + Kokkos::parallel_for(Kokkos::RangePolicy(0,nglocal),*this); + } else { + double *fix_vector = ifix->vector_grid; + if (nglocal && !fix_vector) + error->all(FLERR,"Fix used by fix ave/grid/kk does not produce " + "a per-grid vector"); + + // the tally was last written on the device this step (zeroed and/or + // accumulated by the kernels above), so mark it device-modified and + // pull it to the host before adding to it, then push the sum back; + // otherwise the host add would operate on stale values and the + // sync_device() would clobber the on-device accumulation + + tally_to_host(); + for (int i = 0; i < nglocal; i++) + tally[i][k] += fix_vector[i]; + } + + } else { + jm1 = j - 1; + int device_ok = fixKKBase && fixKKBase->d_array_grid.data() && + (int) fixKKBase->d_array_grid.extent(0) >= nglocal && + (int) fixKKBase->d_array_grid.extent(1) > jm1; + + if (device_ok) { + tally_to_device(); + d_fix_array = fixKKBase->d_array_grid; + Kokkos::parallel_for(Kokkos::RangePolicy(0,nglocal),*this); + } else { + double **fix_array = ifix->array_grid; + if (nglocal && !fix_array) + error->all(FLERR,"Fix used by fix ave/grid/kk does not produce " + "a per-grid array"); + + tally_to_host(); + for (int i = 0; i < nglocal; i++) + tally[i][k] += fix_array[i][jm1]; + } + } // evaluate grid-style variable, sum values to Kth column of tally array } else if (which[m] == VARIABLE) { k = umap[m][0]; + + // compute_grid() with sumflag = 1 adds into the host tally, so the host + // copy has to be current first: earlier values in this same command + // accumulate on the device, and k_tally is not marked device-modified + // until after this loop. Without the pull-down the sum would read + // stale values and the push-back would clobber the device work + + tally_to_host(); input->variable->compute_grid(n,&tally[0][k],ntotal,1); - k_tally.modify_host(); - k_tally.sync_device(); // access custom attribute @@ -283,6 +348,7 @@ void FixAveGridKokkos::end_of_step() auto gridKK = (GridKokkos*) grid; gridKK->sync(Device,CUSTOM_MASK); + tally_to_device(); k = umap[m][0]; if (j == 0) { @@ -374,6 +440,11 @@ void FixAveGridKokkos::end_of_step() } } + // a trailing run of host values may have left the tally on the host, so + // push it down before claiming the device below + + tally_to_device(); + // the tally array was accumulated on the device this step // mark it modified so the host copy is refreshed if grid cells later // migrate (the migration hooks pack/unpack the host tally) @@ -616,6 +687,60 @@ void FixAveGridKokkos::add_grid_one() pergrid_modify(Host); } +/* ---------------------------------------------------------------------- + bring the per-grid output up to date on the device. + + The migration hooks above leave vector_grid/array_grid current on the host + only, and end_of_step() is what normally refreshes the device. Another + Kokkos style can read this fix's output before then: fix adapt invokes + compute lambda/grid from AdaptGrid::candidates_coarsen(), which is after + refinement has already migrated cells, and that compute launches a kernel + straight on d_array_grid. Without this the kernel reads the values the + cells held before the migration. + + PERGRIDSURF keeps its per-cell arrays in host memory and allocates no + device views, so there is nothing to publish. +------------------------------------------------------------------------- */ + +void FixAveGridKokkos::sync_pergrid_device_kokkos() +{ + if (flavor == PERGRIDSURF) return; + + pergrid_sync(Device); + + // the consumer reads the cached handles, so re-point them at the views + // pergrid_sync() just refreshed + + if (nvalues == 1) d_vector_grid = k_vector_grid.view_device(); + else d_array_grid = k_array_grid.view_device(); +} + +/* ---------------------------------------------------------------------- + move k_tally to one side, only when it is not already there. the value + loop in end_of_step() mixes device values (computes, fixes that publish a + device view, custom attributes) with host ones (a fix with no device view, + a grid variable); transferring at the crossings collapses a run of host + values into a single round trip +------------------------------------------------------------------------- */ + +void FixAveGridKokkos::tally_to_host() +{ + if (tally_on_host) return; + k_tally.modify_device(); + k_tally.sync_host(); + tally_on_host = 1; +} + +/* ---------------------------------------------------------------------- */ + +void FixAveGridKokkos::tally_to_device() +{ + if (!tally_on_host) return; + k_tally.modify_host(); + k_tally.sync_device(); + tally_on_host = 0; +} + /* ---------------------------------------------------------------------- sync/modify the per-grid dual views: the tally array plus whichever output array (vector_grid or array_grid) is in use diff --git a/src/KOKKOS/fix_ave_grid_kokkos.h b/src/KOKKOS/fix_ave_grid_kokkos.h index 2a539875a..dd4f253c1 100644 --- a/src/KOKKOS/fix_ave_grid_kokkos.h +++ b/src/KOKKOS/fix_ave_grid_kokkos.h @@ -51,6 +51,18 @@ class FixAveGridKokkos : public FixAveGrid, public KokkosBase { void copy_grid_one(int, int); void add_grid_one(); + void sync_pergrid_device_kokkos(); + + // k_tally crosses between host and device only when the side in use + // actually changes. Each host-side value used to pull the whole + // nglocal x ntotal tally down and push it straight back, so N host + // values in one command cost N full round trips where one does + // (see the value loop in end_of_step()) + + int tally_on_host; + void tally_to_host(); + void tally_to_device(); + KOKKOS_INLINE_FUNCTION void operator()(TagFixAveGrid_Zero_group_vector, const int&) const; diff --git a/src/KOKKOS/fix_ave_histo_kokkos.cpp b/src/KOKKOS/fix_ave_histo_kokkos.cpp index 2322f9a29..f3ca3fbc6 100644 --- a/src/KOKKOS/fix_ave_histo_kokkos.cpp +++ b/src/KOKKOS/fix_ave_histo_kokkos.cpp @@ -65,7 +65,14 @@ FixAveHistoKokkos::FixAveHistoKokkos(SPARTA *spa, int narg, char **arg) : k_stats.resize(4); d_stats = k_stats.view_device(); - memory->destroy(bin); + // FixAveHisto allocates bin with new double[] (fix_ave_histo.cpp:417) and + // releases it with delete[] (:482). memory->destroy() routes to sfree() + // and so free()s a new[] allocation -- undefined behaviour, which + // AddressSanitizer reports as alloc-dealloc-mismatch. The Kokkos build + // replaces the array with a dual view, so release it the way the host + // allocated it before grow_kokkos() takes over + + delete [] bin; bin = NULL; memoryKK->grow_kokkos(k_bin, bin, nbins, "ave/histo:bin"); d_bin = k_bin.view_device(); @@ -155,10 +162,11 @@ void FixAveHistoKokkos::end_of_step() for (int i = 0; i < nbins; i++) k_bin.view_host()(i) = 0.0; k_bin.modify_host(); k_bin.sync_device(); + + minmax_type(minmax).init(minmax); } - minmax_type::value_type minmax; - minmax_type reducer(minmax); + minmax_type reducer(mm_scratch); // accumulate results of computes,fixes,variables to local copy // compute/fix/variable may invoke computes so wrap with clear/add @@ -263,6 +271,9 @@ void FixAveHistoKokkos::end_of_step() if (!fix->kokkos_flag) error->all(FLERR,"Cannot (yet) use non-Kokkos fixes with fix ave/histo/kk"); KokkosBase* fixKKBase = dynamic_cast(fix); + // a fix keeps its per-grid output between invocations and grid migration + // can leave it current on the host alone, so ask for the device copy + if (fixKKBase) fixKKBase->sync_pergrid_device_kokkos(); if (kind == GLOBAL && mode == SCALAR) { if (j == 0) { @@ -482,6 +493,7 @@ void FixAveHistoKokkos::bin_vector( minmax_type& reducer, int n, double *values, int stride) { + minmax_reset(); using FixKokkosDetails::mirror_view_from_raw_host_array; this->stride = stride; @@ -489,6 +501,7 @@ void FixAveHistoKokkos::bin_vector( auto policy = Kokkos::RangePolicy(0, n); Kokkos::parallel_reduce(policy, *this, reducer); + minmax_fold(); } /* ---------------------------------------------------------------------- @@ -499,6 +512,7 @@ void FixAveHistoKokkos::bin_particles( minmax_type& reducer, int attribute, int index) { + minmax_reset(); using Kokkos::RangePolicy; this->index = index; @@ -552,6 +566,7 @@ void FixAveHistoKokkos::bin_particles( Kokkos::parallel_reduce(policy, *this, reducer); } } + minmax_fold(); } /* ---------------------------------------------------------------------- @@ -561,6 +576,7 @@ void FixAveHistoKokkos::bin_particles( minmax_type& reducer, double *values, int stride) { + minmax_reset(); using Kokkos::RangePolicy; using FixKokkosDetails::mirror_view_from_raw_host_array; @@ -598,6 +614,7 @@ void FixAveHistoKokkos::bin_particles( auto policy = RangePolicy(0, n); Kokkos::parallel_reduce(policy, *this, reducer); } + minmax_fold(); } /* ---------------------------------------------------------------------- @@ -607,6 +624,7 @@ void FixAveHistoKokkos::bin_grid_cells( minmax_type& reducer, DAT::t_float_1d_strided d_vec) { + minmax_reset(); using Kokkos::RangePolicy; using FixKokkosDetails::mirror_view_from_raw_host_array; @@ -622,6 +640,7 @@ void FixAveHistoKokkos::bin_grid_cells( auto policy = RangePolicy(0, n); Kokkos::parallel_reduce(policy, *this, reducer); } + minmax_fold(); } diff --git a/src/KOKKOS/fix_ave_histo_kokkos.h b/src/KOKKOS/fix_ave_histo_kokkos.h index 7eb9f59ce..b116a02c2 100644 --- a/src/KOKKOS/fix_ave_histo_kokkos.h +++ b/src/KOKKOS/fix_ave_histo_kokkos.h @@ -69,6 +69,29 @@ class FixAveHistoKokkos : public FixAveHisto typedef Kokkos::MinMax minmax_type; typedef minmax_type::value_type mm_value_type; + // min/max accumulate across the whole Nrepeat window, so this cannot be + // a local in end_of_step(): it is reset only at irepeat == 0, exactly + // as the host resets stats[2]/stats[3] (fix_ave_histo.cpp:549-553) + + mm_value_type minmax; + + // ... and the reducer must NOT be bound to it. Kokkos::parallel_reduce() + // writes its result through the reducer's bound reference and calls + // init() on that reference first (Kokkos_Serial_Parallel_Range.hpp: the + // result pointer is passed straight to reducer.init()), so binding to + // minmax would reset the running window on every launch and leave only + // the last kernel's extrema. Reduce into this scratch value instead and + // fold it in: minmax_reset() before a launch, minmax_fold() after + + mm_value_type mm_scratch; + + void minmax_reset() { minmax_type(mm_scratch).init(mm_scratch); } + + void minmax_fold() { + if (mm_scratch.min_val < minmax.min_val) minmax.min_val = mm_scratch.min_val; + if (mm_scratch.max_val > minmax.max_val) minmax.max_val = mm_scratch.max_val; + } + FixAveHistoKokkos(class SPARTA *, int, char **); virtual ~FixAveHistoKokkos(); void init(); @@ -199,7 +222,14 @@ class FixAveHistoKokkos : public FixAveHisto void bin_one(mm_value_type& mm_v, double value) const { - bin_one(mm_v, value, 1.); + // fix ave/histo/weight carries a single scalar weight for scalar-mode + // inputs, computed in calculate_weights(). The host injects it by + // overriding the single-value bin_one + // (fix_ave_histo_weight.cpp:310-312), but the call sites here are in + // the base class and this overload is not virtual, so read the member + // directly. weightflag is 0 for plain ave/histo, leaving weight 1 + + bin_one(mm_v, value, weightflag ? weight : 1.0); } }; diff --git a/src/KOKKOS/fix_ave_histo_weight_kokkos.cpp b/src/KOKKOS/fix_ave_histo_weight_kokkos.cpp index 7dde85694..41a60c539 100644 --- a/src/KOKKOS/fix_ave_histo_weight_kokkos.cpp +++ b/src/KOKKOS/fix_ave_histo_weight_kokkos.cpp @@ -223,6 +223,9 @@ void FixAveHistoWeightKokkos::calculate_weights() if (!fix->kokkos_flag) error->all(FLERR,"Cannot (yet) use non-Kokkos fixes with fix ave/histo/weight/kk"); KokkosBase* fixKKBase = dynamic_cast(fix); + // a fix keeps its per-grid output between invocations and grid migration + // can leave it current on the host alone, so ask for the device copy + if (fixKKBase) fixKKBase->sync_pergrid_device_kokkos(); if (kind == GLOBAL && mode == SCALAR) { if (j == 0) { @@ -304,6 +307,7 @@ void FixAveHistoWeightKokkos::bin_vector( minmax_type& reducer, int n, double *values, int stride) { + minmax_reset(); using FixKokkosDetails::mirror_view_from_raw_host_array; this->stride = stride; @@ -311,6 +315,7 @@ void FixAveHistoWeightKokkos::bin_vector( auto policy = Kokkos::RangePolicy(0, n); Kokkos::parallel_reduce(policy, *this, reducer); + minmax_fold(); } /* ---------------------------------------------------------------------- @@ -321,6 +326,7 @@ void FixAveHistoWeightKokkos::bin_particles( minmax_type& reducer, int attribute, int index) { + minmax_reset(); using Kokkos::RangePolicy; using FixKokkosDetails::mirror_view_from_raw_host_array; @@ -375,6 +381,7 @@ void FixAveHistoWeightKokkos::bin_particles( Kokkos::parallel_reduce(policy, *this, reducer); } } + minmax_fold(); } /* ---------------------------------------------------------------------- @@ -384,6 +391,7 @@ void FixAveHistoWeightKokkos::bin_particles( minmax_type& reducer, double *values, int stride) { + minmax_reset(); using Kokkos::RangePolicy; using FixKokkosDetails::mirror_view_from_raw_host_array; @@ -421,6 +429,7 @@ void FixAveHistoWeightKokkos::bin_particles( auto policy = RangePolicy(0, n); Kokkos::parallel_reduce(policy, *this, reducer); } + minmax_fold(); } /* ---------------------------------------------------------------------- @@ -430,6 +439,7 @@ void FixAveHistoWeightKokkos::bin_grid_cells( minmax_type& reducer, DAT::t_float_1d_strided d_vec) { + minmax_reset(); using Kokkos::RangePolicy; using FixKokkosDetails::mirror_view_from_raw_host_array; @@ -446,6 +456,7 @@ void FixAveHistoWeightKokkos::bin_grid_cells( auto policy = RangePolicy(0, n); Kokkos::parallel_reduce(policy, *this, reducer); } + minmax_fold(); } /* ------------------------------------------------------------------------- */ diff --git a/src/KOKKOS/fix_dt_reset_kokkos.cpp b/src/KOKKOS/fix_dt_reset_kokkos.cpp index bee6bfbf1..940a350e9 100644 --- a/src/KOKKOS/fix_dt_reset_kokkos.cpp +++ b/src/KOKKOS/fix_dt_reset_kokkos.cpp @@ -93,6 +93,9 @@ void FixDtResetKokkos::end_of_step() if (!fstep->kokkos_flag) error->all(FLERR,"Cannot (yet) use non-Kokkos fixes with fix dt/reset/kk"); KokkosBase* computeKKBase = dynamic_cast(fstep); + // a fix keeps its per-grid output between invocations and grid migration + // can leave it current on the host alone, so ask for the device copy + if (computeKKBase) computeKKBase->sync_pergrid_device_kokkos(); if (step_index == 0) copy_gridstep(computeKKBase->d_vector_grid,nglocal); else { diff --git a/src/KOKKOS/fix_emit_face_file_kokkos.cpp b/src/KOKKOS/fix_emit_face_file_kokkos.cpp new file mode 100644 index 000000000..5e8571983 --- /dev/null +++ b/src/KOKKOS/fix_emit_face_file_kokkos.cpp @@ -0,0 +1,1040 @@ +/* ---------------------------------------------------------------------- + SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer + http://sparta.github.io + Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov + Sandia National Laboratories + + Copyright (2014) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level SPARTA directory. +------------------------------------------------------------------------- */ + +#include "stdlib.h" +#include "string.h" +#include "fix_emit_face_file_kokkos.h" +#include "update.h" +#include "domain.h" +#include "region.h" +#include "grid.h" +#include "surf.h" +#include "particle.h" +#include "mixture.h" +#include "comm.h" +#include "modify.h" +#include "random_knuth.h" +#include "math_const.h" +#include "memory_kokkos.h" +#include "error.h" +#include "kokkos_type.h" +#include "particle_kokkos.h" +#include "grid_kokkos.h" +#include "fix_emit_kokkos.h" +#include "sparta_masks.h" +#include "Kokkos_Random.hpp" + +using namespace SPARTA_NS; +using namespace MathConst; + +enum{XLO,XHI,YLO,YHI,ZLO,ZHI,INTERIOR}; // same as Domain +enum{PERIODIC,OUTFLOW,REFLECT,SURFACE,AXISYM}; // same as Domain +enum{UNKNOWN,OUTSIDE,INSIDE,OVERLAP}; // same as Grid +enum{PKEEP,PINSERT,PDONE,PDISCARD,PENTRY,PEXIT,PSURF}; // several files +enum{NCHILD,NPARENT,NUNKNOWN,NPBCHILD,NPBPARENT,NPBUNKNOWN,NBOUND}; // Grid +enum{NRHO,TEMP_THERMAL,TEMP_ROT,TEMP_VIB,VX,VY,VZ,PRESS,SPECIES}; +enum{NOSUBSONIC,PTBOTH,PONLY}; + +#define DELTATASK 256 +#define TEMPLIMIT 1.0e5 + +/* ---------------------------------------------------------------------- + insert particles on a boundary face, with per-face flow properties + interpolated from a file, on the device +------------------------------------------------------------------------- */ + +FixEmitFaceFileKokkos::FixEmitFaceFileKokkos(SPARTA *sparta, int narg, char **arg) : + FixEmitFaceFile(sparta, narg, arg), + rand_pool(12345 + comm->me +#ifdef SPARTA_KOKKOS_EXACT + , sparta +#endif + ), + particle_kk_copy(sparta) +{ + kokkos_flag = 1; + execution_space = Device; + + // this fix runs at START_OF_STEP. ModifyKokkos::start_of_step() calls + // ParticleKokkos::sync(execution_space,datamask_read) before the fix and + // ::modify(execution_space,datamask_modify) after it. Both masks are + // EMPTY here because this fix does all of its own syncing: it needs + // particles on the device only when subsonic, and it appends to the + // particle list itself (grow + modify(Device,PARTICLE_MASK)) inside + // perform_task(). Declaring PARTICLE_MASK here would force a + // host<->device round trip of the whole particle list on every step of + // every run that uses this fix. + + datamask_read = EMPTY_MASK; + datamask_modify = EMPTY_MASK; + + region_flag = 0; + nregion_token = 0; + axisymmetric = 0; + dt_step = 0.0; + boltz = 0.0; + plist_descending = 0; +} + +/* ---------------------------------------------------------------------- */ + +FixEmitFaceFileKokkos::~FixEmitFaceFileKokkos() +{ + if (copymode) return; + +#ifdef SPARTA_KOKKOS_EXACT + rand_pool.destroy(); +#endif + + // the per-task vectors point into Kokkos DualViews, not into new[] memory, + // so hide them from ~FixEmitFaceFile(), which would delete[] them + + if (tasks) { + for (int i = 0; i < ntaskmax; i++) { + tasks[i].ntargetsp = NULL; + tasks[i].vscale = NULL; + tasks[i].fraction = NULL; + tasks[i].cummulative = NULL; + } + } + + // tasks itself is the host half of k_tasks, so it must not reach + // memory->sfree() in ~FixEmitFaceFile(). zero ntaskmax as well: unlike + // ~FixEmitFace(), ~FixEmitFaceFile() has no "if (tasks)" guard around its + // delete[] loop, so a NULL tasks with a nonzero ntaskmax would fault + + tasks = NULL; + ntaskmax = 0; +} + +/* ---------------------------------------------------------------------- */ + +void FixEmitFaceFileKokkos::init() +{ + // fix emit/face/file supports a one-pass and a two-pass insertion loop. + // only the two-pass one draws every task's insertion count before it + // generates any particle, which is the order the Kokkos kernel pair is + // forced into: the offset scan has to know all the counts before the + // candidate arrays can be sized. So under SPARTA_KOKKOS_EXACT, where + // the whole point is a bit-for-bit match against the non-Kokkos run, + // refuse to run without it. Outside SPARTA_KOKKOS_EXACT stay silent, + // the same as fix emit/face/kk: the random streams differ anyway. + +#ifdef SPARTA_KOKKOS_EXACT + if (!twopass) + error->all(FLERR,"Fix emit/face/file/kk requires the twopass keyword " + "under SPARTA_KOKKOS_EXACT: without it the Kokkos and " + "non-Kokkos runs consume random numbers in a different order " + "and will not produce the same particles"); +#endif + + // pull anything the device wrote (subsonic updates the tasks and vscale) + // back to the host before host-side code reads or overwrites it, and + // before any of these DualViews is replaced below + + k_tasks.sync_host(); + if (perspecies) k_ntargetsp.sync_host(); + k_vscale.sync_host(); + k_cummulative.sync_host(); + k_fraction.sync_host(); + + // FixEmitFaceFile::init() delete[]s and re-new[]s tasks[i].fraction, + // .cummulative, .vscale and .ntargetsp for the ntask tasks left over from + // the previous run, because the mixture's species count may have changed. + // Ours are not new[] memory -- they alias rows of the DualViews -- so + // that loop must not run on them. FixEmitFaceFile has no + // realloc_nspecies() hook the way FixEmitFace does, so do the equivalent + // here instead: + // - pick up the new species count (init() sets the same value again) + // - resize the DualViews and re-aim every Task pointer at their rows + // - zero ntask, which makes those two base loops no-ops. Nothing else + // in init() reads ntask, and FixEmit::create_tasks() -- called at the + // end of init() -- zeroes it itself and rebuilds the whole list, so + // this changes nothing for the host. + + nspecies = particle->mixture[imix]->nspecies; + realloc_species_views(); + ntask = 0; + + FixEmitFaceFile::init(); + + // create_tasks() ran inside init() and wrote the task values, plus the + // per-task fraction/cummulative/vscale/ntargetsp rows, on the host + + k_tasks.modify_host(); + if (perspecies) k_ntargetsp.modify_host(); + k_vscale.modify_host(); + k_cummulative.modify_host(); + k_fraction.modify_host(); + +#ifdef SPARTA_KOKKOS_EXACT + rand_pool.init(random); +#endif + + // domain->axisymmetric is not reachable from a device kernel + + axisymmetric = domain->axisymmetric; + + // mixture species indices, the only mixture-wide array the kernels need. + // vscale/fraction/cummulative are per-task for this style, since they are + // interpolated from the file per face, so there is no mixture-wide copy + + k_mspecies = DAT::tdual_int_1d("emit/face/file:mspecies",nspecies); + d_mspecies = k_mspecies.view_device(); + + auto h_mspecies = k_mspecies.view_host(); + for (int isp = 0; isp < nspecies; isp++) + h_mspecies(isp) = particle->mixture[imix]->species[isp]; + + k_mspecies.modify_host(); + // the region is fixed for the run; flatten it once here rather than + // rebuilding and re-uploading the token stream in perform_task() + + flatten_region(); + +} + +/* ---------------------------------------------------------------------- + create tasks for all grid cells + the interpolation of the file mesh onto each cell face is host-only setup: + it happens here, once per grid change, never per step. the task values + it writes land directly in the host half of the DualViews (see + realloc_species_views()), so all this override has to do is mark them + dirty +------------------------------------------------------------------------- */ + +void FixEmitFaceFileKokkos::create_tasks() +{ + k_tasks.sync_host(); + if (perspecies) k_ntargetsp.sync_host(); + k_vscale.sync_host(); + k_cummulative.sync_host(); + k_fraction.sync_host(); + + FixEmitFaceFile::create_tasks(); + + k_tasks.modify_host(); + if (perspecies) k_ntargetsp.modify_host(); + k_vscale.modify_host(); + k_cummulative.modify_host(); + k_fraction.modify_host(); +} + +/* ---------------------------------------------------------------------- + flatten the region to a device-resident postfix token stream, so the + emit kernel needs no virtual dispatch and no typed copy per region style. + the stream carries each sub-region's interior/exterior sense and the + composite's own, so nothing else needs to be passed along. + see region_prim_kokkos.h + + a region is fixed for the duration of a run -- Region exposes no move or + rotate, and "region" is an input command -- so this runs from init() + rather than from perform_task(), which would rebuild the stream on the + host and re-upload it every step +------------------------------------------------------------------------- */ + +void FixEmitFaceFileKokkos::flatten_region() +{ + region_flag = 0; + nregion_token = 0; + if (region) { + KokkosBase* region_kkbase = dynamic_cast(region); + if (!region->kokkos_flag || !region_kkbase) + error->all(FLERR,"KOKKOS package does not (yet) support chosen region style"); + nregion_token = region_kkbase->flatten_region_kokkos(k_region_tokens); + if (nregion_token <= 0) + error->all(FLERR,"KOKKOS package does not (yet) support chosen region style"); + d_region_tokens = k_region_tokens.view_device(); + region_flag = 1; + } +} + +/* ---------------------------------------------------------------------- + insert particles in grid cells with faces touching the inflow boundary + always two-pass: count kernel -> offset scan -> generate kernel -> + compaction +------------------------------------------------------------------------- */ + +void FixEmitFaceFileKokkos::perform_task() +{ + // the non-Kokkos perform_task_*() read update->dt into a local, leaving the + // class member dt (frozen at init) for subsonic_inflow(). keep that + // split so a run with fix dt/reset behaves identically + + dt_step = update->dt; + + // face geometry is fix-wide here, not per task as in fix emit/face, so + // hoist it into locals the compaction lambda can capture by value + // (a device lambda may not capture this) + + auto l_dimension = this->dimension; + auto l_ndim = this->ndim; + auto l_pdim = this->pdim; + auto l_qdim = this->qdim; + const double l_normal_ndim = normal[ndim]; + + // if subsonic, re-compute particle inflow counts for each task + // also computes current per-task temp_thermal and vstream + + if (subsonic) subsonic_inflow(); + + // insert particles for each task = cell/face pair + // ntarget/ninsert is either perspecies or for all species + + // copy needed task data to device + + if (perspecies) k_ntargetsp.sync_device(); + else k_tasks.sync_device(); + + auto ninsert_dim1 = perspecies ? nspecies : 1; + if (d_ninsert.extent(0) < ntask * ninsert_dim1) + d_ninsert = DAT::t_int_1d("emit/face/file:ninsert", ntask * ninsert_dim1); + + copymode = 1; + Kokkos::parallel_for(Kokkos::RangePolicy(0,ntask),*this); + copymode = 0; + + int ncands; + d_task2cand = offset_scan(d_ninsert, ncands); + + if (ncands == 0) return; + + // for one particle: + // x = random position on subset of face that overlaps with file grid + // v = randomized thermal velocity + vstream + // first stage: normal dimension (ndim) + // second stage: parallel dimensions (pdim,qdim) + + // double while loop until randomized particle velocity meets 2 criteria + // inner do-while loop: + // v = vstream-component + vthermal is into simulation box + // see Bird 1994, p 425 + // outer do-while loop: + // shift Maxwellian distribution by stream velocity component + // see Bird 1994, p 259, eq 12.5 + + if (d_x.extent(0) < ncands || d_x.extent(1) < l_dimension) + d_x = DAT::t_float_2d("emit/face/file:x", ncands, l_dimension); + + if (d_task.extent(0) < ncands) { + d_beta_un = DAT::t_float_1d("emit/face/file:beta_un", ncands); + d_theta = DAT::t_float_1d("emit/face/file:theta", ncands); + d_vr = DAT::t_float_1d("emit/face/file:vr", ncands); + d_erot = DAT::t_float_1d("emit/face/file:erot", ncands); + d_evib = DAT::t_float_1d("emit/face/file:evib", ncands); + d_dtremain = DAT::t_float_1d("emit/face/file:dtremain", ncands); + d_id = DAT::t_int_1d("emit/face/file:id", ncands); + d_isp = DAT::t_int_1d("emit/face/file:isp", ncands); + d_task = DAT::t_int_1d("emit/face/file:task", ncands); + d_keep = DAT::t_int_1d("emit/face/file:keep", ncands); + } + Kokkos::deep_copy(d_keep,0); // needs to be initialized with zeros + + auto ld_x = d_x ; + auto ld_beta_un = d_beta_un ; + auto ld_theta = d_theta ; + auto ld_vr = d_vr ; + auto ld_erot = d_erot ; + auto ld_evib = d_evib ; + auto ld_dtremain = d_dtremain; + auto ld_id = d_id ; + auto ld_isp = d_isp ; + auto ld_task = d_task ; + auto ld_keep = d_keep ; + + // copy needed task data to device + // fraction/cummulative/vscale are per-task for this style, so all of them + // ride along with the tasks rather than coming from the mixture + + k_tasks.sync_device(); + if (perspecies) k_ntargetsp.sync_device(); + k_vscale.sync_device(); + k_cummulative.sync_device(); + + auto ld_tasks = d_tasks; + auto ld_vscale = d_vscale; + + k_mspecies.sync_device(); + auto ld_mspecies = d_mspecies; + + ParticleKokkos* particle_kk = ((ParticleKokkos*)particle); + particle_kk->update_class_variables(); + particle_kk_copy.copy(particle_kk); + + // flatten the region to a device-resident postfix token stream, so the + // kernel below needs no virtual dispatch and no typed copy per region + // style. the stream carries each sub-region's interior/exterior sense + // and the composite's own, so nothing else needs to be passed along. + // see region_prim_kokkos.h + + + int nsingle_reduce = 0; + copymode = 1; + Kokkos::parallel_reduce(Kokkos::RangePolicy(0,ntask),*this,nsingle_reduce); + copymode = 0; + nsingle += nsingle_reduce; + + int nnew; + auto ld_cands2new = offset_scan(d_keep, nnew); + + auto particleKK = dynamic_cast(particle); + auto nlocal_before = particleKK->nlocal; + particleKK->grow(nnew); + particleKK->sync(SPARTA_NS::Device, PARTICLE_MASK); + auto ld_particles = particleKK->k_particles.view_device(); + + Kokkos::parallel_for(ncands, SPARTA_LAMBDA(int cand) { + if (!ld_keep(cand)) return; + + auto i = ld_task(cand); + Task task_i = ld_tasks(i); + + const int pcell = task_i.pcell; + double *vstream = task_i.vstream; + + auto isp = ld_isp(cand); + auto vscale_val = ld_vscale(i, isp); + auto ispecies = ld_mspecies(isp); + + double x[3]; + for (int d = 0; d < l_dimension; ++d) x[d] = ld_x(cand, d); + for (int d = l_dimension; d < 3; ++d) x[d] = 0; + + auto beta_un = ld_beta_un(cand); + auto theta = ld_theta(cand); + auto vr = ld_vr(cand); + auto erot = ld_erot(cand); + auto evib = ld_evib(cand); + auto id = ld_id(cand); + auto dtremain = ld_dtremain(cand); + + double v[3]; + v[l_ndim] = beta_un*vscale_val*l_normal_ndim + vstream[l_ndim]; + v[l_pdim] = vr * sin(theta) + vstream[l_pdim]; + v[l_qdim] = vr * cos(theta) + vstream[l_qdim]; + + auto inew = ld_cands2new(cand); + auto ilocal = nlocal_before + inew; + + ParticleKokkos::add_particle_kokkos(ld_particles,ilocal, + id,ispecies,pcell,x,v,erot,evib); + + ld_particles(ilocal).flag = PINSERT; + ld_particles(ilocal).dtremain = dtremain; + }); + + particleKK->nlocal = nlocal_before + nnew; + particleKK->modify(SPARTA_NS::Device, PARTICLE_MASK); + particleKK->zero_custom_kokkos(nlocal_before,particleKK->nlocal); + + // custom per-particle attributes are still a host-side callback + + if (modify->n_update_custom) { + auto h_keep = Kokkos::create_mirror_view(d_keep); + auto h_task = Kokkos::create_mirror_view(d_task); + Kokkos::deep_copy(h_keep, d_keep); + Kokkos::deep_copy(h_task, d_task); + + // copy needed task data to host + + k_tasks.sync_host(); + + auto h_cands2new = Kokkos::create_mirror_view(ld_cands2new); + Kokkos::deep_copy(h_cands2new, ld_cands2new); + + for (int cand = 0; cand < ncands; ++cand) { + if (!h_keep(cand)) continue; + + auto task = h_task(cand); + + auto temp_thermal = tasks[task].temp_thermal; + auto temp_rot = tasks[task].temp_rot; + auto temp_vib = tasks[task].temp_vib; + auto vstream = tasks[task].vstream; + + auto inew = h_cands2new(cand); + auto ilocal = nlocal_before + inew; + + modify->update_custom(ilocal,temp_thermal,temp_rot,temp_vib,vstream); + } + } +} + +/* ---------------------------------------------------------------------- + # of particles to insert for each task + fix emit/face/file has no "n" and no "modulate" option, so unlike + fix emit/face this is just ntarget + a uniform deviate + this is the first of the two passes: every count is drawn here, before + any particle is generated, which is what perform_task_twopass() on the + host mirrors +------------------------------------------------------------------------- */ + +KOKKOS_INLINE_FUNCTION +void FixEmitFaceFileKokkos::operator()(TagFixEmitFaceFile_ninsert, const int &i) const +{ + rand_type rand_gen = rand_pool.get_state(); + + if (perspecies) { + for (int isp = 0; isp < nspecies; isp++) { + const double ntarget = d_ntargetsp(i,isp) + rand_gen.drand(); + d_ninsert(i * nspecies + isp) = static_cast (ntarget); + } + } else { + const double ntarget = d_tasks(i).ntarget + rand_gen.drand(); + d_ninsert(i) = static_cast (ntarget); + } + + rand_pool.free_state(rand_gen); +} + +/* ---------------------------------------------------------------------- */ + +KOKKOS_INLINE_FUNCTION +void FixEmitFaceFileKokkos::operator()(TagFixEmitFaceFile_perform_task, + const int &i, int &nsingle_reduce) const +{ + rand_type rand_gen = rand_pool.get_state(); + + Task task_i = d_tasks(i); + + double *lo = task_i.lo; + double *hi = task_i.hi; + double *vstream = task_i.vstream; + + const double temp_rot = task_i.temp_rot; + const double temp_vib = task_i.temp_vib; + + // normal is fix-wide for this style, not per task + + const double indot = vstream[0]*normal[0] + vstream[1]*normal[1] + + vstream[2]*normal[2]; + + if (perspecies) { + for (int isp = 0; isp < nspecies; isp++) { + const int ispecies = d_mspecies[isp]; + + // per-task vscale, not the mixture's: the file can set a per-face + // temperature, which interpolate() folded into the task + + const double vscale_val = d_vscale(i,isp); + const double scosine = indot / vscale_val; + + const int ninsert = d_ninsert(i * nspecies + isp); + const int start = d_task2cand(i * nspecies + isp); + + int nactual = 0; + for (int m = 0; m < ninsert; m++) { + const int cand = start + m; + + double x[3]; + x[0] = lo[0] + rand_gen.drand() * (hi[0]-lo[0]); + if (axisymmetric) + x[1] = sqrt(lo[1]*lo[1] + + rand_gen.drand() * (hi[1]*hi[1]-lo[1]*lo[1])); + else x[1] = lo[1] + rand_gen.drand() * (hi[1]-lo[1]); + if (dimension == 3) x[2] = lo[2] + rand_gen.drand() * (hi[2]-lo[2]); + else x[2] = 0.0; + + // region_flag must be tested first: with no region there is no token + // stream, and region_match_kk() of an empty stream rejects + // everything + + if (region_flag && + !region_match_kk(d_region_tokens,nregion_token, + x[0],x[1],x[2])) continue; + + nactual++; + d_keep(cand) = 1; + d_task(cand) = i; + d_isp(cand) = isp; + for (int d = 0; d < dimension; ++d) d_x(cand, d) = x[d]; + + double beta_un, normalized_distbn_fn; + do { + do beta_un = (6.0*rand_gen.drand() - 3.0); + while (beta_un + scosine < 0.0); + normalized_distbn_fn = 2.0 * (beta_un + scosine) / + (scosine + sqrt(scosine*scosine + 2.0)) * + exp(0.5 + (0.5*scosine)*(scosine-sqrt(scosine*scosine + 2.0)) - + beta_un*beta_un); + } while (normalized_distbn_fn < rand_gen.drand()); + + d_beta_un(cand) = beta_un; + + d_theta(cand) = MY_2PI * rand_gen.drand(); + d_vr(cand) = vscale_val * sqrt(-log(rand_gen.drand())); + d_erot(cand) = particle_kk_copy.obj.erot(ispecies,temp_rot,rand_gen); + d_evib(cand) = particle_kk_copy.obj.evib(ispecies,temp_vib,rand_gen); + d_id(cand) = MAXSMALLINT*rand_gen.drand(); + d_dtremain(cand) = dt_step * rand_gen.drand(); + } + + nsingle_reduce += nactual; + } + + } else { + const int ninsert = d_ninsert(i); + const int start = d_task2cand(i); + + int nactual = 0; + for (int m = 0; m < ninsert; m++) { + const int cand = start + m; + + // per-task cummulative, not the mixture's: the file can set per-face + // species fractions, which interpolate() folded into the task + + const double rn = rand_gen.drand(); + int isp = 0; + while (d_cummulative(i,isp) < rn) isp++; + + const int ispecies = d_mspecies[isp]; + const double vscale_val = d_vscale(i,isp); + const double scosine = indot / vscale_val; + + double x[3]; + x[0] = lo[0] + rand_gen.drand() * (hi[0]-lo[0]); + if (axisymmetric) + x[1] = sqrt(lo[1]*lo[1] + + rand_gen.drand() * (hi[1]*hi[1]-lo[1]*lo[1])); + else x[1] = lo[1] + rand_gen.drand() * (hi[1]-lo[1]); + if (dimension == 3) x[2] = lo[2] + rand_gen.drand() * (hi[2]-lo[2]); + else x[2] = 0.0; + + // region_flag must be tested first: with no region there is no token + // stream, and region_match_kk() of an empty stream rejects everything + + if (region_flag && + !region_match_kk(d_region_tokens,nregion_token, + x[0],x[1],x[2])) continue; + + nactual++; + d_keep(cand) = 1; + d_task(cand) = i; + d_isp(cand) = isp; + for (int d = 0; d < dimension; ++d) d_x(cand, d) = x[d]; + + double beta_un, normalized_distbn_fn; + do { + do beta_un = (6.0*rand_gen.drand() - 3.0); + while (beta_un + scosine < 0.0); + normalized_distbn_fn = 2.0 * (beta_un + scosine) / + (scosine + sqrt(scosine*scosine + 2.0)) * + exp(0.5 + (0.5*scosine)*(scosine-sqrt(scosine*scosine + 2.0)) - + beta_un*beta_un); + } while (normalized_distbn_fn < rand_gen.drand()); + + d_beta_un(cand) = beta_un; + + d_theta(cand) = MY_2PI * rand_gen.drand(); + d_vr(cand) = vscale_val * sqrt(-log(rand_gen.drand())); + d_erot(cand) = particle_kk_copy.obj.erot(ispecies,temp_rot,rand_gen); + d_evib(cand) = particle_kk_copy.obj.evib(ispecies,temp_vib,rand_gen); + d_id(cand) = MAXSMALLINT*rand_gen.drand(); + d_dtremain(cand) = dt_step * rand_gen.drand(); + } + + nsingle_reduce += nactual; + } + + rand_pool.free_state(rand_gen); +} + +/* ---------------------------------------------------------------------- + recalculate task properties based on subsonic BC + subsonic is enabled by a "press" column in the input file, so it is a + property of the file, not of a keyword +------------------------------------------------------------------------- */ + +void FixEmitFaceFileKokkos::subsonic_inflow() +{ + // for grid cells that are part of tasks: + // calculate local nrho, vstream, and thermal temperature + // if needed sort particles for grid cells with tasks + + subsonic_sort(); + subsonic_grid(); + + // recalculate particle insertion counts for each task + // vscale here is recomputed from the per-task temp_thermal, exactly as the + // non-Kokkos subsonic_inflow() does -- it does NOT read tasks[i].vscale + + ParticleKokkos* particle_kk = (ParticleKokkos*) particle; + particle_kk->sync(Device,SPECIES_MASK); + d_species_all = particle_kk->k_species.view_device(); + + GridKokkos* grid_kk = (GridKokkos*) grid; + grid_kk->sync(Device,CINFO_MASK); + d_cinfo = grid_kk->k_cinfo.view_device(); + + k_tasks.sync_device(); + if (perspecies) k_ntargetsp.sync_device(); + k_mspecies.sync_device(); + k_fraction.sync_device(); + + boltz = update->boltz; + + copymode = 1; + Kokkos::parallel_for(Kokkos::RangePolicy(0,ntask),*this); + copymode = 0; + + k_tasks.modify_device(); + if (perspecies) k_ntargetsp.modify_device(); + + // release references to reduce memory use + + d_species_all = t_species_1d(); + d_cinfo = {}; +} + +/* ---------------------------------------------------------------------- */ + +KOKKOS_INLINE_FUNCTION +void FixEmitFaceFileKokkos::operator()(TagFixEmitFaceFile_subsonic_inflow, + const int &i) const +{ + double *vstream = d_tasks(i).vstream; + const double indot = vstream[0]*normal[0] + vstream[1]*normal[1] + + vstream[2]*normal[2]; + + const double area = d_tasks(i).area; + const double nrho = d_tasks(i).nrho; + const double temp_thermal = d_tasks(i).temp_thermal; + const int icell = d_tasks(i).icell; + + // fraction is per task here, since the file can set per-face species + // fractions -- fix emit/face uses the mixture-wide vector instead + + double ntarget = 0.0; + for (int isp = 0; isp < nspecies; isp++) { + const double mass = d_species_all[d_mspecies[isp]].mass; + const double vscale = sqrt(2.0 * boltz * temp_thermal / mass); + double ntargetsp = mol_inflow_kokkos(indot,vscale,d_fraction(i,isp)); + ntargetsp *= nrho*area*dt / fnum; + ntargetsp /= d_cinfo[icell].weight; + ntarget += ntargetsp; + if (perspecies) d_ntargetsp(i,isp) = ntargetsp; + } + + d_tasks(i).ntarget = ntarget; + if (ntarget >= MAXSMALLINT) + Kokkos::abort("Fix emit/face/file subsonic insertion count " + "exceeds 32-bit int"); +} + +/* ---------------------------------------------------------------------- + sort particles into grid cells on device + the non-Kokkos FixEmitFaceFile::subsonic_sort() builds its own per-cell + linked list (Grid::ChildInfo first/count + Particle::next) for the + "active" cells only. on device we instead reuse the compressed per-cell + particle lists ParticleKokkos::sort_kokkos() builds for collisions + (GridKokkos d_plist/d_cellcount), which cover every cell. the moment + sums below are the only consumer, and they are per-cell, so covering + extra cells costs nothing but the sort itself. + this is also why the host-side activecell[] bookkeeping and the + active_current flag are simply not used by the Kokkos path +------------------------------------------------------------------------- */ + +void FixEmitFaceFileKokkos::subsonic_sort() +{ + ParticleKokkos* particle_kk = (ParticleKokkos*) particle; + + // sorted_kk mirrors the host Particle::sorted flag the non-Kokkos path + // tests before calling subsonic_sort(). Record it BEFORE sorting: when + // the non-Kokkos code has to build the list itself it pushes each + // particle on the head, so the list comes out in DEcreasing particle + // index; an already-sorted list (built by Particle::sort()'s reverse + // loop) comes out in INcreasing index. d_plist is always increasing, + // so remember which order to walk it in. + + plist_descending = !particle_kk->sorted_kk; + if (!particle_kk->sorted_kk) particle_kk->sort_kokkos(); +} + +/* ---------------------------------------------------------------------- + compute number density, thermal temperature, stream velocity + only for grid cells associated with a task + first compute for grid cells, then adjust due to boundary conditions +------------------------------------------------------------------------- */ + +void FixEmitFaceFileKokkos::subsonic_grid() +{ + ParticleKokkos* particle_kk = (ParticleKokkos*) particle; + particle_kk->sync(Device,PARTICLE_MASK|SPECIES_MASK); + d_particles = particle_kk->k_particles.view_device(); + d_species_all = particle_kk->k_species.view_device(); + + // refresh particle_kk_copy since particle data structures may + // have changed since the last copy, e.g. by sort or grow + + particle_kk->update_class_variables(); + particle_kk_copy.copy(particle_kk); + + GridKokkos* grid_kk = (GridKokkos*) grid; + grid_kk->sync(Device,CINFO_MASK); + d_cinfo = grid_kk->k_cinfo.view_device(); + d_plist = grid_kk->d_plist; + d_cellcount = grid_kk->d_cellcount; + + k_tasks.sync_device(); + if (subsonic_style == PONLY) { + k_vscale.sync_device(); + k_mspecies.sync_device(); + } + + boltz = update->boltz; + + // only track max thermal temp until the one-time warning has fired + // avoids a per-step device->host fence once subsonic_warning is set + + if (!subsonic_warning) { + if (d_tempmax.data() == nullptr) + d_tempmax = DAT::t_float_scalar("emit/face/file:tempmax"); + Kokkos::deep_copy(d_tempmax,0.0); + } + + copymode = 1; + Kokkos::parallel_for(Kokkos::RangePolicy(0,ntask),*this); + copymode = 0; + + k_tasks.modify_device(); + if (subsonic_style == PONLY) k_vscale.modify_device(); + + // release references to reduce memory use + + d_particles = t_particle_1d(); + d_species_all = t_species_1d(); + d_plist = {}; + d_cellcount = {}; + d_cinfo = {}; + + // test if any task has invalid thermal temperature for first time + + if (!subsonic_warning) { + double tempmax = 0.0; + Kokkos::deep_copy(tempmax,d_tempmax); + int temp_exceed_flag = 0; + if (tempmax > TEMPLIMIT) temp_exceed_flag = 1; + subsonic_warning = subsonic_temperature_check(temp_exceed_flag,tempmax); + } +} + +/* ---------------------------------------------------------------------- */ + +KOKKOS_INLINE_FUNCTION +void FixEmitFaceFileKokkos::operator()(TagFixEmitFaceFile_subsonic_grid, + const int &i) const +{ + const int icell = d_tasks(i).pcell; + const int np = d_cellcount(icell); + + // accumulate needed per-particle quantities + // mv = mass*velocity terms, masstot = total mass + // gamma = rotational/tranlational DOFs + + double mv[4]; + mv[0] = mv[1] = mv[2] = mv[3] = 0.0; + double masstot = 0.0; + double gamma = 0.0; + + // d_plist orders particles by increasing index. The non-Kokkos path walks + // whichever linked list is current: the one Particle::sort() builds (head = + // lowest index, so INcreasing order) when the particles were already + // sorted, else the one subsonic_sort() builds itself (head = highest index, + // so DEcreasing order). For SPARTA_KOKKOS_EXACT match that order so the + // per-cell moment sums are bit-identical (serial, single thread, host). + +#ifdef SPARTA_KOKKOS_EXACT + const int nbeg = plist_descending ? np-1 : 0; + const int nend = plist_descending ? -1 : np; + const int ninc = plist_descending ? -1 : 1; + for (int n = nbeg; n != nend; n += ninc) { +#else + for (int n = 0; n < np; n++) { +#endif + const int ip = d_plist(icell,n); + const int ispecies = d_particles[ip].ispecies; + const double mass = d_species_all[ispecies].mass; + const double *v = d_particles[ip].v; + mv[0] += mass*v[0]; + mv[1] += mass*v[1]; + mv[2] += mass*v[2]; + mv[3] += mass * (v[0]*v[0]+v[1]*v[1]+v[2]*v[2]); + masstot += mass; + gamma += 1.0 + 2.0 / (3.0 + d_species_all[ispecies].rotdof); + } + + // compute/store nrho, 3 temps, vstream for task + // also vscale for PONLY + // if sound speed = 0.0 due to <= 1 particle in cell or + // all particles having COM velocity, set via mixture properties + + double *vstream = d_tasks(i).vstream; + if (np) { + vstream[0] = mv[0] / masstot; + vstream[1] = mv[1] / masstot; + vstream[2] = mv[2] / masstot; + } else vstream[0] = vstream[1] = vstream[2] = 0.0; + + // press is the file-interpolated pressure for THIS task, which is what + // distinguishes this style from fix emit/face's global psubsonic + + const double press = d_tasks(i).press; + + double temp_thermal_cell; + + if (subsonic_style == PTBOTH) { + d_tasks(i).nrho = press / (boltz * d_tasks(i).temp_thermal); + temp_thermal_cell = d_tasks(i).temp_thermal; + + } else { + const double nrho_cell = np * fnum / d_cinfo[icell].volume; + const double massrho_cell = masstot * fnum / d_cinfo[icell].volume; + if (np > 1) { + const double ke = mv[3]/np - + (mv[0]*mv[0] + mv[1]*mv[1] + mv[2]*mv[2])/np/masstot; + temp_thermal_cell = tprefactor * ke; + } else temp_thermal_cell = temp_thermal_mix; + + const double press_cell = nrho_cell * boltz * temp_thermal_cell; + double soundspeed_cell; + if (np) { + const double mass_cell = masstot / np; + const double gamma_cell = gamma / np; + soundspeed_cell = sqrt(gamma_cell*boltz*temp_thermal_cell / mass_cell); + } else soundspeed_cell = soundspeed_mixture; + + d_tasks(i).nrho = nrho_cell + + (press - press_cell) / (soundspeed_cell*soundspeed_cell); + temp_thermal_cell = press / (boltz * d_tasks(i).nrho); + if (!subsonic_warning && temp_thermal_cell > TEMPLIMIT) + Kokkos::atomic_max(&d_tempmax(),temp_thermal_cell); + + // the non-Kokkos code guards this update with massrho_cell*soundspeed + // > 0.0 as well as np, unlike fix emit/face. keep the extra guard + + if (np && massrho_cell*soundspeed_cell > 0.0) { + const double sign = normal[ndim]; + vstream[ndim] += sign * + (press - press_cell) / (massrho_cell*soundspeed_cell); + } + + for (int m = 0; m < nspecies; m++) { + const int ispecies = d_mspecies[m]; + d_vscale(i,m) = sqrt(2.0 * boltz * temp_thermal_cell / + d_species_all[ispecies].mass); + } + } + + d_tasks(i).temp_thermal = temp_thermal_cell; + d_tasks(i).temp_rot = d_tasks(i).temp_vib = temp_thermal_cell; +} + +/* ---------------------------------------------------------------------- + grow task list +------------------------------------------------------------------------- */ + +void FixEmitFaceFileKokkos::grow_task() +{ + ntaskmax += DELTATASK; + + k_tasks.sync_host(); + k_tasks.modify_host(); // force resize on host + memoryKK->grow_kokkos(k_tasks,tasks,ntaskmax,"emit/face/file:tasks"); + d_tasks = k_tasks.view_device(); + + // allocate vectors in each new task or set to NULL + // ntargetsp is only used for perspecies, exactly as in the non-Kokkos code + + if (perspecies) { + k_ntargetsp.sync_host(); + k_ntargetsp.modify_host(); // force resize on host + k_ntargetsp.resize(ntaskmax,nspecies); + d_ntargetsp = k_ntargetsp.view_device(); + for (int i = 0; i < ntaskmax; i++) + tasks[i].ntargetsp = &k_ntargetsp.view_host()(i,0); + } else { + for (int i = 0; i < ntaskmax; i++) + tasks[i].ntargetsp = NULL; + } + + // fraction/cummulative/vscale are per-task for every run of this style, + // subsonic or not, because the file can vary them face by face. so + // unlike fix emit/face they are always allocated, and the Task pointers + // the host interpolate() writes through are aimed at their host rows + + k_vscale.sync_host(); + k_vscale.modify_host(); + k_vscale.resize(ntaskmax,nspecies); + d_vscale = k_vscale.view_device(); + + k_cummulative.sync_host(); + k_cummulative.modify_host(); + k_cummulative.resize(ntaskmax,nspecies); + d_cummulative = k_cummulative.view_device(); + + k_fraction.sync_host(); + k_fraction.modify_host(); + k_fraction.resize(ntaskmax,nspecies); + d_fraction = k_fraction.view_device(); + + for (int i = 0; i < ntaskmax; i++) { + tasks[i].vscale = &k_vscale.view_host()(i,0); + tasks[i].cummulative = &k_cummulative.view_host()(i,0); + tasks[i].fraction = &k_fraction.view_host()(i,0); + } +} + +/* ---------------------------------------------------------------------- + (re)allocate the per-task species arrays and re-aim the Task pointers + called from init(), in place of the realloc_nspecies() hook that + FixEmitFace has and FixEmitFaceFile does not: the mixture's species + count may have changed since the last run, and the rows have to be the + right width before init() -> create_tasks() -> interpolate() starts + writing through the Task pointers + the old contents are dropped, exactly as the base class's delete[]/new[] + pair drops them; create_tasks() rewrites every task from the file mesh +------------------------------------------------------------------------- */ + +void FixEmitFaceFileKokkos::realloc_species_views() +{ + if (perspecies) { + k_ntargetsp = DAT::tdual_float_2d_lr("emit/face/file:ntargetsp", + ntaskmax,nspecies); + d_ntargetsp = k_ntargetsp.view_device(); + for (int i = 0; i < ntaskmax; i++) + tasks[i].ntargetsp = &k_ntargetsp.view_host()(i,0); + } else { + k_ntargetsp = DAT::tdual_float_2d_lr(); + d_ntargetsp = DAT::t_float_2d_lr(); + for (int i = 0; i < ntaskmax; i++) + tasks[i].ntargetsp = NULL; + } + + k_vscale = DAT::tdual_float_2d_lr("emit/face/file:vscale", + ntaskmax,nspecies); + k_cummulative = DAT::tdual_float_2d_lr("emit/face/file:cummulative", + ntaskmax,nspecies); + k_fraction = DAT::tdual_float_2d_lr("emit/face/file:fraction", + ntaskmax,nspecies); + + d_vscale = k_vscale.view_device(); + d_cummulative = k_cummulative.view_device(); + d_fraction = k_fraction.view_device(); + + for (int i = 0; i < ntaskmax; i++) { + tasks[i].vscale = &k_vscale.view_host()(i,0); + tasks[i].cummulative = &k_cummulative.view_host()(i,0); + tasks[i].fraction = &k_fraction.view_host()(i,0); + } +} diff --git a/src/KOKKOS/fix_emit_face_file_kokkos.h b/src/KOKKOS/fix_emit_face_file_kokkos.h new file mode 100644 index 000000000..54f37369d --- /dev/null +++ b/src/KOKKOS/fix_emit_face_file_kokkos.h @@ -0,0 +1,185 @@ +/* ---------------------------------------------------------------------- + SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer + http://sparta.github.io + Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov + Sandia National Laboratories + + Copyright (2014) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level SPARTA directory. +------------------------------------------------------------------------- */ + +#ifdef FIX_CLASS + +FixStyle(emit/face/file/kk,FixEmitFaceFileKokkos) + +#else + +#ifndef SPARTA_FIX_EMIT_FACE_FILE_KOKKOS_H +#define SPARTA_FIX_EMIT_FACE_FILE_KOKKOS_H + +#include "fix_emit_face_file.h" +#include "rand_pool_wrap.h" +#include "kokkos_base.h" +#include "kokkos_copy.h" +#include "particle_kokkos.h" +#include "region_prim_kokkos.h" + +namespace SPARTA_NS { + +struct TagFixEmitFaceFile_ninsert{}; +struct TagFixEmitFaceFile_perform_task{}; +struct TagFixEmitFaceFile_subsonic_inflow{}; +struct TagFixEmitFaceFile_subsonic_grid{}; + +class FixEmitFaceFileKokkos : public FixEmitFaceFile { + public: + typedef int value_type; + + FixEmitFaceFileKokkos(class SPARTA *, int, char **); + ~FixEmitFaceFileKokkos() override; + void init() override; + + // the Kokkos path is always two-pass: the count scan has to know every + // task's insertion count before candidate arrays can be sized. so both + // entry points land on the same kernel pair, exactly as fix emit/face/kk + // does (fix_emit_face_kokkos.h:46-47) + + void flatten_region(); + void perform_task() override; + void perform_task_twopass() override { perform_task(); } + + KOKKOS_INLINE_FUNCTION + void operator()(TagFixEmitFaceFile_ninsert, const int&) const; + + KOKKOS_INLINE_FUNCTION + void operator()(TagFixEmitFaceFile_perform_task, const int&, int&) const; + + KOKKOS_INLINE_FUNCTION + void operator()(TagFixEmitFaceFile_subsonic_inflow, const int&) const; + + KOKKOS_INLINE_FUNCTION + void operator()(TagFixEmitFaceFile_subsonic_grid, const int&) const; + +#ifndef SPARTA_KOKKOS_EXACT + Kokkos::Random_XorShift64_Pool rand_pool; + typedef typename Kokkos::Random_XorShift64_Pool::generator_type rand_type; + + //Kokkos::Random_XorShift1024_Pool rand_pool; + //typedef typename Kokkos::Random_XorShift1024_Pool::generator_type rand_type; +#else + RandPoolWrap rand_pool; + typedef RandWrap rand_type; +#endif + + private: + int region_flag; + int axisymmetric; // copy of domain->axisymmetric, needed on device + // FixEmitFaceFile, unlike FixEmitFace, keeps no + // copy of it + double boltz; + double dt_step; // update->dt for the current step, used for dtremain + // kept separate from the base member dt, which the + // non-Kokkos code freezes at init() and uses for + // the subsonic ntarget recalculation + + KKCopy particle_kk_copy; + + // region flattened to a device-resident postfix token stream, so the + // insertion kernel needs no virtual dispatch and no typed copy per + // region style. the stream carries each sub-region's interior/exterior + // sense and the composite's own, so nothing else needs to be passed + // along. region_flag says whether there is a region at all -- + // nregion_token and d_region_tokens are only meaningful when it is 1. + // see region_prim_kokkos.h + + tdual_region_token_1d k_region_tokens; + t_region_token_1d d_region_tokens; + int nregion_token; + + typedef Kokkos::DualView tdual_task_1d; + typedef tdual_task_1d::t_dev t_task_1d; + tdual_task_1d k_tasks; + t_task_1d d_tasks; + + // per-task, per-species arrays. + // unlike fix emit/face, fix emit/face/file carries a per-task fraction, + // cummulative and vscale ALWAYS -- they are interpolated from the file + // per face, not taken mixture-wide -- so all three are allocated + // unconditionally, and the kernels index them (i,isp) rather than (isp). + // the host Task::fraction/cummulative/vscale/ntargetsp pointers are aimed + // at the host rows of these DualViews by realloc_species_views() and + // grow_task(). that is what lets the unmodified host interpolate() go + // on writing through those pointers: the file mesh setup stays entirely + // host-side and is flattened once, at task-build time, not per step. + + DAT::tdual_float_2d_lr k_ntargetsp; // # of mols to insert for each species + DAT::tdual_float_2d_lr k_vscale; // vscale for each species + DAT::tdual_float_2d_lr k_cummulative; // cummulative fraction for each species + DAT::tdual_float_2d_lr k_fraction; // fraction for each species + DAT::t_float_2d_lr d_ntargetsp; + DAT::t_float_2d_lr d_vscale; + DAT::t_float_2d_lr d_cummulative; + DAT::t_float_2d_lr d_fraction; + + Kokkos::View d_ninsert; + DAT::t_int_1d d_task2cand; + + DAT::t_float_2d d_x; + DAT::t_float_1d d_beta_un; + DAT::t_float_1d d_theta; + DAT::t_float_1d d_vr; + DAT::t_float_1d d_erot; + DAT::t_float_1d d_evib; + DAT::t_float_1d d_dtremain; + DAT::t_int_1d d_id; + DAT::t_int_1d d_isp; + DAT::t_int_1d d_task; + Kokkos::View d_keep; // won't compile with DAT::t_int_1d type + + DAT::tdual_int_1d k_mspecies; // species indices of mixture + DAT::t_int_1d d_mspecies; + + // data structs for subsonic emission + + t_particle_1d d_particles; + t_species_1d d_species_all; // all particle species (mass, rotdof) + t_cinfo_1d d_cinfo; + DAT::t_int_2d d_plist; + DAT::t_int_1d d_cellcount; + DAT::t_float_scalar d_tempmax; + int plist_descending; // 1 if the host walks d_plist high index -> low + + void create_tasks() override; + void grow_task() override; + + void subsonic_inflow() override; + void subsonic_sort() override; + void subsonic_grid() override; + + // (re)size the per-task species DualViews and re-aim every Task pointer at + // their host rows. FixEmitFaceFile has no realloc_nspecies() hook the + // way FixEmitFace does, so this is called explicitly from init(), before + // the base init() reaches create_tasks() + + void realloc_species_views(); +}; + +} + +#endif +#endif + +/* ERROR/WARNING messages: + +E: Fix emit/face/file/kk requires the twopass keyword under SPARTA_KOKKOS_EXACT + +The Kokkos insertion loop draws every task's insertion count before it +generates any particle. The non-Kokkos one-pass loop interleaves the two. +Only the non-Kokkos twopass keyword consumes random numbers in the same +order, so only then can the two runs produce the same particles. + +*/ diff --git a/src/KOKKOS/fix_emit_face_kokkos.cpp b/src/KOKKOS/fix_emit_face_kokkos.cpp index 5f249589d..1a19b8599 100644 --- a/src/KOKKOS/fix_emit_face_kokkos.cpp +++ b/src/KOKKOS/fix_emit_face_kokkos.cpp @@ -65,11 +65,7 @@ FixEmitFaceKokkos::FixEmitFaceKokkos(SPARTA *sparta, int narg, char **arg) : , sparta #endif ), - particle_kk_copy(sparta), - regblock_kk_copy(sparta), - regcylinder_kk_copy(sparta), - regplane_kk_copy(sparta), - regsphere_kk_copy(sparta) + particle_kk_copy(sparta) { kokkos_flag = 1; execution_space = Device; @@ -77,6 +73,7 @@ FixEmitFaceKokkos::FixEmitFaceKokkos(SPARTA *sparta, int narg, char **arg) : datamask_modify = EMPTY_MASK; region_flag = 0; + nregion_token = 0; } /* ---------------------------------------------------------------------- */ @@ -141,6 +138,11 @@ void FixEmitFaceKokkos::init() k_cummulative.modify_host(); k_mspecies.modify_host(); k_fraction.modify_host(); + // the region is fixed for the run; flatten it once here rather than + // rebuilding and re-uploading the token stream in perform_task() + + flatten_region(); + } /* ---------------------------------------------------------------------- @@ -161,6 +163,35 @@ void FixEmitFaceKokkos::create_tasks() if (subsonic_style == PONLY) k_vscale.modify_host(); } +/* ---------------------------------------------------------------------- + flatten the region to a device-resident postfix token stream, so the + emit kernel needs no virtual dispatch and no typed copy per region style. + the stream carries each sub-region's interior/exterior sense and the + composite's own, so nothing else needs to be passed along. + see region_prim_kokkos.h + + a region is fixed for the duration of a run -- Region exposes no move or + rotate, and "region" is an input command -- so this runs from init() + rather than from perform_task(), which would rebuild the stream on the + host and re-upload it every step +------------------------------------------------------------------------- */ + +void FixEmitFaceKokkos::flatten_region() +{ + region_flag = 0; + nregion_token = 0; + if (region) { + KokkosBase* region_kkbase = dynamic_cast(region); + if (!region->kokkos_flag || !region_kkbase) + error->all(FLERR,"KOKKOS package does not (yet) support chosen region style"); + nregion_token = region_kkbase->flatten_region_kokkos(k_region_tokens); + if (nregion_token <= 0) + error->all(FLERR,"KOKKOS package does not (yet) support chosen region style"); + d_region_tokens = k_region_tokens.view_device(); + region_flag = 1; + } +} + /* ---------------------------------------------------------------------- */ void FixEmitFaceKokkos::perform_task() @@ -186,9 +217,14 @@ void FixEmitFaceKokkos::perform_task() // ntarget/ninsert is either perspecies or for all species // copy needed task data to device + // the kernels below read d_tasks whether or not perspecies is set, so tasks + // is synced unconditionally and ntargetsp in addition, the same shape as + // the second copy further down this routine. The if/else this replaces + // left tasks unsynced under perspecies and was only harmless because that + // later copy covered it + k_tasks.sync_device(); if (perspecies) k_ntargetsp.sync_device(); - else k_tasks.sync_device(); auto ninsert_dim1 = perspecies ? nspecies : 1; if (d_ninsert.extent(0) < ntask * ninsert_dim1) @@ -267,31 +303,12 @@ void FixEmitFaceKokkos::perform_task() particle_kk->update_class_variables(); particle_kk_copy.copy(particle_kk); - if (region && !region->kokkos_flag) - error->all(FLERR,"KOKKOS package does not (yet) support chosen region style"); + // flatten the region to a device-resident postfix token stream, so the + // kernel below needs no virtual dispatch and no typed copy per region + // style. the stream carries each sub-region's interior/exterior sense + // and the composite's own, so nothing else needs to be passed along. + // see region_prim_kokkos.h - region_flag = 0; - if (region) { - if (strstr(region->style,"block") != NULL) { - RegBlockKokkos* region_kk = ((RegBlockKokkos*)region); - regblock_kk_copy.copy(region_kk); - region_flag = 1; - } else if (strstr(region->style,"cylinder") != NULL) { - RegCylinderKokkos* region_kk = ((RegCylinderKokkos*)region); - regcylinder_kk_copy.copy(region_kk); - region_flag = 2; - } else if (strstr(region->style,"plane") != NULL) { - RegPlaneKokkos* region_kk = ((RegPlaneKokkos*)region); - regplane_kk_copy.copy(region_kk); - region_flag = 3; - } else if (strstr(region->style,"sphere") != NULL) { - RegSphereKokkos* region_kk = ((RegSphereKokkos*)region); - regsphere_kk_copy.copy(region_kk); - region_flag = 4; - } else { - error->all(FLERR,"KOKKOS package does not (yet) support chosen region style"); - } - } int nsingle_reduce = 0; copymode = 1; @@ -462,15 +479,9 @@ void FixEmitFaceKokkos::operator()(TagFixEmitFace_perform_task, const int &i, in if (dimension == 3) x[2] = lo[2] + rand_gen.drand() * (hi[2]-lo[2]); else x[2] = 0.0; - if (region_flag == 1) { - if (!regblock_kk_copy.obj.match_kokkos(x[0], x[1], x[2])) continue; - } else if (region_flag == 2) { - if (!regcylinder_kk_copy.obj.match_kokkos(x[0], x[1], x[2])) continue; - } else if (region_flag == 3) { - if (!regplane_kk_copy.obj.match_kokkos(x[0], x[1], x[2])) continue; - } else if (region_flag == 4) { - if (!regsphere_kk_copy.obj.match_kokkos(x[0], x[1], x[2])) continue; - } + if (region_flag && + !region_match_kk(d_region_tokens,nregion_token, + x[0],x[1],x[2])) continue; nactual++; d_keep(cand) = 1; @@ -523,15 +534,9 @@ void FixEmitFaceKokkos::operator()(TagFixEmitFace_perform_task, const int &i, in if (dimension == 3) x[2] = lo[2] + rand_gen.drand() * (hi[2]-lo[2]); else x[2] = 0.0; - if (region_flag == 1) { - if (!regblock_kk_copy.obj.match_kokkos(x[0], x[1], x[2])) continue; - } else if (region_flag == 2) { - if (!regcylinder_kk_copy.obj.match_kokkos(x[0], x[1], x[2])) continue; - } else if (region_flag == 3) { - if (!regplane_kk_copy.obj.match_kokkos(x[0], x[1], x[2])) continue; - } else if (region_flag == 4) { - if (!regsphere_kk_copy.obj.match_kokkos(x[0], x[1], x[2])) continue; - } + if (region_flag && + !region_match_kk(d_region_tokens,nregion_token, + x[0],x[1],x[2])) continue; nactual++; d_keep(cand) = 1; diff --git a/src/KOKKOS/fix_emit_face_kokkos.h b/src/KOKKOS/fix_emit_face_kokkos.h index 3258bf4e7..78650a028 100644 --- a/src/KOKKOS/fix_emit_face_kokkos.h +++ b/src/KOKKOS/fix_emit_face_kokkos.h @@ -26,15 +26,10 @@ FixStyle(emit/face/kk,FixEmitFaceKokkos) #include "kokkos_base.h" #include "kokkos_copy.h" #include "particle_kokkos.h" -#include "region_block_kokkos.h" -#include "region_cylinder_kokkos.h" -#include "region_plane_kokkos.h" -#include "region_sphere_kokkos.h" +#include "region_prim_kokkos.h" namespace SPARTA_NS { -#define KOKKOS_MAX_REGION_PER_TYPE 2 -#define KOKKOS_MAX_TOT_REGION 10 struct TagFixEmitFace_ninsert{}; struct TagFixEmitFace_perform_task{}; @@ -48,6 +43,7 @@ class FixEmitFaceKokkos : public FixEmitFace { FixEmitFaceKokkos(class SPARTA *, int, char **); ~FixEmitFaceKokkos() override; void init() override; + void flatten_region(); void perform_task() override; void perform_task_twopass() override { perform_task(); } @@ -80,10 +76,14 @@ class FixEmitFaceKokkos : public FixEmitFace { double boltz,temp_thermal_mix; KKCopy particle_kk_copy; - KKCopy regblock_kk_copy; - KKCopy regcylinder_kk_copy; - KKCopy regplane_kk_copy; - KKCopy regsphere_kk_copy; + // region flattened to a device-resident postfix token stream; replaces + // the per-style KKCopy members and the caps that went with them. + // region_flag says whether there is a region at all -- nregion_token and + // d_region_tokens are only meaningful when it is 1 + + tdual_region_token_1d k_region_tokens; + t_region_token_1d d_region_tokens; + int nregion_token; typedef Kokkos::DualView tdual_task_1d; typedef tdual_task_1d::t_dev t_task_1d; diff --git a/src/KOKKOS/fix_emit_surf_kokkos.cpp b/src/KOKKOS/fix_emit_surf_kokkos.cpp index 4541a65d9..4c5a2d31a 100644 --- a/src/KOKKOS/fix_emit_surf_kokkos.cpp +++ b/src/KOKKOS/fix_emit_surf_kokkos.cpp @@ -15,6 +15,7 @@ #include "stdlib.h" #include "string.h" #include "fix_emit_surf_kokkos.h" +#include "compute_isurf_grid_kokkos.h" #include "update.h" #include "domain.h" #include "region.h" @@ -41,6 +42,44 @@ #include "Kokkos_Random.hpp" using namespace SPARTA_NS; + +#ifndef SPARTA_KOKKOS_FIXED_LISTS +namespace { + + // the surf-tally computes this fix drives are held in one runtime-sized + // device byte buffer instead of a fixed KKCopy array, so their number is + // not capped. the three steps mirror KKCopy exactly: size the buffer, + // blit each live object's bytes into its slot and mark the image a copy + // so its destructor frees nothing, then push the buffer to the device. + // this is the same code UpdateKokkos uses for its own tally lists + // (update_kokkos.cpp:94-108); it is duplicated rather than shared + // because both live in anonymous namespaces in their own translation + // units, and the whole point is that neither is a class member + + void slist_buf_grow(DAT::tdual_char_1d &k, int n, size_t bytes) + { + const size_t need = (size_t) n * bytes; + if (k.view_host().extent(0) >= need) return; + k = DAT::tdual_char_1d("fix_emit_surf:slist_surf",need); + } + + template + void slist_buf_blit(DAT::tdual_char_1d &k, int slot, T *obj) + { + char *dst = k.view_host().data() + (size_t) slot*sizeof(T); + memcpy((void*) dst, (const void*) obj, sizeof(T)); + ((T *) dst)->copy = 1; + } + + void slist_buf_sync(DAT::tdual_char_1d &k, DAT::t_char_1d &d) + { + if (k.view_device().extent(0) == 0) return; + k.modify_host(); + k.sync_device(); + d = k.view_device(); + } +} +#endif using namespace MathConst; enum{PKEEP,PINSERT,PDONE,PDISCARD,PENTRY,PEXIT,PSURF}; // several files @@ -65,13 +104,11 @@ FixEmitSurfKokkos::FixEmitSurfKokkos(SPARTA *sparta, int narg, char **arg) : , sparta #endif ), - particle_kk_copy(sparta), - slist_active_copy{VAL_2(KKCopy(sparta))}, - tmp_compute_surf_kk(sparta), - regblock_kk_copy(sparta), - regcylinder_kk_copy(sparta), - regplane_kk_copy(sparta), - regsphere_kk_copy(sparta) + particle_kk_copy(sparta) +#ifdef SPARTA_KOKKOS_FIXED_LISTS + , slist_active_copy{VAL_2(KKCopy(sparta))} + , tmp_compute_surf_kk(sparta) +#endif { kokkos_flag = 1; execution_space = Device; @@ -79,6 +116,7 @@ FixEmitSurfKokkos::FixEmitSurfKokkos(SPARTA *sparta, int narg, char **arg) : datamask_modify = EMPTY_MASK; region_flag = 0; + nregion_token = 0; } @@ -92,6 +130,14 @@ FixEmitSurfKokkos::~FixEmitSurfKokkos() rand_pool.destroy(); #endif + // the loop below reads and writes the host side of k_tasks, which the + // emission kernels leave claimed for the device. path and fracarea are host + // allocations the device never writes, so the host values are the right ones + // and no copy back is wanted here; say so rather than reading a side the + // coherence state calls stale. + + k_tasks.clear_sync_state(); + for (int i = 0; i < ntaskmax; i++) { tasks[i].ntargetsp = NULL; tasks[i].vscale = NULL; @@ -165,6 +211,11 @@ void FixEmitSurfKokkos::init() k_mspecies.modify_host(); k_fraction.modify_host(); + // the region is fixed for the run; flatten it once here rather than + // rebuilding and re-uploading the token stream in perform_task() + + flatten_region(); + } /* ---------------------------------------------------------------------- @@ -240,6 +291,35 @@ void FixEmitSurfKokkos::create_tasks() k_fracarea.modify_host(); } +/* ---------------------------------------------------------------------- + flatten the region to a device-resident postfix token stream, so the + emit kernel needs no virtual dispatch and no typed copy per region style. + the stream carries each sub-region's interior/exterior sense and the + composite's own, so nothing else needs to be passed along. + see region_prim_kokkos.h + + a region is fixed for the duration of a run -- Region exposes no move or + rotate, and "region" is an input command -- so this runs from init() + rather than from perform_task(), which would rebuild the stream on the + host and re-upload it every step +------------------------------------------------------------------------- */ + +void FixEmitSurfKokkos::flatten_region() +{ + region_flag = 0; + nregion_token = 0; + if (region) { + KokkosBase* region_kkbase = dynamic_cast(region); + if (!region->kokkos_flag || !region_kkbase) + error->all(FLERR,"KOKKOS package does not (yet) support chosen region style"); + nregion_token = region_kkbase->flatten_region_kokkos(k_region_tokens); + if (nregion_token <= 0) + error->all(FLERR,"KOKKOS package does not (yet) support chosen region style"); + d_region_tokens = k_region_tokens.view_device(); + region_flag = 1; + } +} + /* ---------------------------------------------------------------------- */ void FixEmitSurfKokkos::perform_task() @@ -276,9 +356,14 @@ void FixEmitSurfKokkos::perform_task() // see Bird 1994, p 259, eq 12.5 // copy needed task data to device + // the kernels below read d_tasks whether or not perspecies is set, so tasks + // is synced unconditionally and ntargetsp in addition, the same shape as + // the second copy further down this routine. The if/else this replaces + // left tasks unsynced under perspecies and was only harmless because that + // later copy covered it + k_tasks.sync_device(); if (perspecies) k_ntargetsp.sync_device(); - else k_tasks.sync_device(); SurfKokkos* surf_kk = (SurfKokkos*) surf; surf_kk->sync(Device,ALL_MASK); @@ -289,19 +374,41 @@ void FixEmitSurfKokkos::perform_task() nsurf_tally = update->nsurf_tally; Compute **slist_active = update->slist_active; +#ifdef SPARTA_KOKKOS_FIXED_LISTS if (nsurf_tally > KOKKOS_MAX_SLIST) - error->all(FLERR,"Kokkos currently only supports two instances of compute surface"); + error->all(FLERR,"Kokkos currently only supports two instances of compute " + "surface with fix emit/surf; rebuild without " + "-DSPARTA_KOKKOS_FIXED_LISTS to lift the limit"); +#else + slist_buf_grow(k_slist_surf,nsurf_tally,sizeof(ComputeSurfKokkos)); +#endif + + // dispatch by dynamic_cast, not by style string: the Kokkos computes are + // registered under their "/kk" names too (isurf/grid/kk et al), so a + // style compare never matches a compute the user typed with the suffix + // and the caller then falls through to an unrelated error message. + // see the same note in UpdateKokkos::setup_surf_tally_copies() for (int i = 0; i < nsurf_tally; i++) { - if (strcmp(slist_active[i]->style,"isurf/grid") == 0) - error->all(FLERR,"Kokkos doesn't yet support compute isurf/grid"); ComputeSurfKokkos* compute_surf_kk = dynamic_cast(slist_active[i]); - if (!compute_surf_kk) - error->all(FLERR,"Kokkos does not (yet) support compute surf/collision/tally or compute surf/reaction/tally"); + if (!compute_surf_kk) { + if (dynamic_cast(slist_active[i])) + error->all(FLERR,"Kokkos does not (yet) support compute isurf/grid " + "with fix emit/surf"); + error->all(FLERR,"Kokkos does not (yet) support this surf tally compute " + "with fix emit/surf; use a Kokkos-enabled surf tally compute " + "(-sf kk)"); + } compute_surf_kk->pre_surf_tally(); +#ifdef SPARTA_KOKKOS_FIXED_LISTS slist_active_copy[i].copy(compute_surf_kk); +#else + slist_buf_blit(k_slist_surf,i,compute_surf_kk); +#endif } +#ifdef SPARTA_KOKKOS_FIXED_LISTS + // every Kokkos functor captures the whole array by value, so the unused // slots must not alias a compute that may be reallocated or deleted while // they still reference count it: point them at a temporary that lives as @@ -309,6 +416,9 @@ void FixEmitSurfKokkos::perform_task() for (int i = nsurf_tally; i < KOKKOS_MAX_SLIST; i++) slist_active_copy[i].copy(&tmp_compute_surf_kk); +#else + slist_buf_sync(k_slist_surf,d_slist_surf); +#endif auto ninsert_dim1 = perspecies ? nspecies : 1; if (d_ninsert.extent(0) < ntask * ninsert_dim1) @@ -365,31 +475,12 @@ void FixEmitSurfKokkos::perform_task() particle_kk->update_class_variables(); particle_kk_copy.copy(particle_kk); - if (region && !region->kokkos_flag) - error->all(FLERR,"KOKKOS package does not (yet) support chosen region style"); + // flatten the region to a device-resident postfix token stream, so the + // kernel below needs no virtual dispatch and no typed copy per region + // style. the stream carries each sub-region's interior/exterior sense + // and the composite's own, so nothing else needs to be passed along. + // see region_prim_kokkos.h - region_flag = 0; - if (region) { - if (strstr(region->style,"block") != NULL) { - RegBlockKokkos* region_kk = ((RegBlockKokkos*)region); - regblock_kk_copy.copy(region_kk); - region_flag = 1; - } else if (strstr(region->style,"cylinder") != NULL) { - RegCylinderKokkos* region_kk = ((RegCylinderKokkos*)region); - regcylinder_kk_copy.copy(region_kk); - region_flag = 2; - } else if (strstr(region->style,"plane") != NULL) { - RegPlaneKokkos* region_kk = ((RegPlaneKokkos*)region); - regplane_kk_copy.copy(region_kk); - region_flag = 3; - } else if (strstr(region->style,"sphere") != NULL) { - RegSphereKokkos* region_kk = ((RegSphereKokkos*)region); - regsphere_kk_copy.copy(region_kk); - region_flag = 4; - } else { - error->all(FLERR,"KOKKOS package does not (yet) support chosen region style"); - } - } int nsingle_reduce = 0; copymode = 1; @@ -578,15 +669,9 @@ void FixEmitSurfKokkos::operator()(TagFixEmitSurf_perform_task, const int &i, in x[2] = p1[2] + alpha*e1[2] + beta*e2[2]; } - if (region_flag == 1) { - if (!regblock_kk_copy.obj.match_kokkos(x[0], x[1], x[2])) continue; - } else if (region_flag == 2) { - if (!regcylinder_kk_copy.obj.match_kokkos(x[0], x[1], x[2])) continue; - } else if (region_flag == 3) { - if (!regplane_kk_copy.obj.match_kokkos(x[0], x[1], x[2])) continue; - } else if (region_flag == 4) { - if (!regsphere_kk_copy.obj.match_kokkos(x[0], x[1], x[2])) continue; - } + if (region_flag && + !region_match_kk(d_region_tokens,nregion_token, + x[0],x[1],x[2])) continue; nactual++; d_keep(cand) = 1; @@ -693,15 +778,9 @@ void FixEmitSurfKokkos::operator()(TagFixEmitSurf_perform_task, const int &i, in x[2] = p1[2] + alpha*e1[2] + beta*e2[2]; } - if (region_flag == 1) { - if (!regblock_kk_copy.obj.match_kokkos(x[0], x[1], x[2])) continue; - } else if (region_flag == 2) { - if (!regcylinder_kk_copy.obj.match_kokkos(x[0], x[1], x[2])) continue; - } else if (region_flag == 3) { - if (!regplane_kk_copy.obj.match_kokkos(x[0], x[1], x[2])) continue; - } else if (region_flag == 4) { - if (!regsphere_kk_copy.obj.match_kokkos(x[0], x[1], x[2])) continue; - } + if (region_flag && + !region_match_kk(d_region_tokens,nregion_token, + x[0],x[1],x[2])) continue; nactual++; d_keep(cand) = 1; @@ -794,7 +873,7 @@ void FixEmitSurfKokkos::operator()(TagFixEmitSurf_insert_particles(dtremain,isurf,pcell,0,NULL,p,NULL); } @@ -921,8 +1000,10 @@ void FixEmitSurfKokkos::subsonic_grid() particle_kk->update_class_variables(); particle_kk_copy.copy(particle_kk); +#ifdef SPARTA_KOKKOS_FIXED_LISTS for (int n = 0; n < KOKKOS_MAX_SLIST; n++) slist_active_copy[n].copy(&tmp_compute_surf_kk); +#endif GridKokkos* grid_kk = (GridKokkos*) grid; grid_kk->sync(Device,CINFO_MASK); @@ -1133,8 +1214,10 @@ void FixEmitSurfKokkos::mflow_grid() particle_kk->update_class_variables(); particle_kk_copy.copy(particle_kk); +#ifdef SPARTA_KOKKOS_FIXED_LISTS for (int n = 0; n < KOKKOS_MAX_SLIST; n++) slist_active_copy[n].copy(&tmp_compute_surf_kk); +#endif GridKokkos* grid_kk = (GridKokkos*) grid; grid_kk->sync(Device,CINFO_MASK); diff --git a/src/KOKKOS/fix_emit_surf_kokkos.h b/src/KOKKOS/fix_emit_surf_kokkos.h index 37a22294b..86e54a56e 100644 --- a/src/KOKKOS/fix_emit_surf_kokkos.h +++ b/src/KOKKOS/fix_emit_surf_kokkos.h @@ -26,15 +26,11 @@ FixStyle(emit/surf/kk,FixEmitSurfKokkos) #include "kokkos_copy.h" #include "particle_kokkos.h" #include "compute_surf_kokkos.h" -#include "region_block_kokkos.h" -#include "region_cylinder_kokkos.h" -#include "region_plane_kokkos.h" -#include "region_sphere_kokkos.h" +#include "kokkos_base.h" +#include "region_prim_kokkos.h" namespace SPARTA_NS { -#define KOKKOS_MAX_SLIST 2 - struct TagFixEmitSurf_ninsert{}; struct TagFixEmitSurf_perform_task{}; struct TagFixEmitSurf_subsonic_inflow{}; @@ -52,6 +48,7 @@ class FixEmitSurfKokkos : public FixEmitSurf { FixEmitSurfKokkos(class SPARTA *, int, char **); ~FixEmitSurfKokkos() override; void init() override; + void flatten_region(); void perform_task() override; void perform_task_twopass() override { perform_task(); } @@ -98,11 +95,27 @@ class FixEmitSurfKokkos : public FixEmitSurf { double acoef,nrho_mflow; KKCopy particle_kk_copy; + // the active compute surf tallies this fix drives, reached on device only + // through FES_SLIST(). Same two layouts as UpdateKokkos's tally lists: + // a fixed KKCopy array held by value in the functor, or one runtime-sized + // device byte buffer with no cap. See kokkos_type.h for why both exist. + +#ifdef SPARTA_KOKKOS_FIXED_LISTS KKCopy slist_active_copy[KOKKOS_MAX_SLIST]; - KKCopy regblock_kk_copy; - KKCopy regcylinder_kk_copy; - KKCopy regplane_kk_copy; - KKCopy regsphere_kk_copy; +#define FES_SLIST(m) slist_active_copy[m].obj +#else + DAT::tdual_char_1d k_slist_surf; + DAT::t_char_1d d_slist_surf; +#define FES_SLIST(m) ((const ComputeSurfKokkos *) d_slist_surf.data())[m] +#endif + // region flattened to a device-resident postfix token stream; replaces + // the per-style KKCopy members and the caps that went with them. + // region_flag says whether there is a region at all -- nregion_token and + // d_region_tokens are only meaningful when it is 1 + + tdual_region_token_1d k_region_tokens; + t_region_token_1d d_region_tokens; + int nregion_token; typedef Kokkos::DualView tdual_task_1d; typedef tdual_task_1d::t_dev t_task_1d; @@ -169,7 +182,11 @@ class FixEmitSurfKokkos : public FixEmitSurf { void subsonic_grid() override; void mflow_grid() override; +#ifdef SPARTA_KOKKOS_FIXED_LISTS + // unused fixed slots must not alias a compute that may be reallocated or + // deleted while they still reference count it ComputeSurfKokkos tmp_compute_surf_kk; +#endif }; } diff --git a/src/KOKKOS/fix_field_grid_kokkos.cpp b/src/KOKKOS/fix_field_grid_kokkos.cpp new file mode 100644 index 000000000..e962b4db8 --- /dev/null +++ b/src/KOKKOS/fix_field_grid_kokkos.cpp @@ -0,0 +1,94 @@ +/* ---------------------------------------------------------------------- + SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer + http://sparta.github.io + Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov + Sandia National Laboratories + + Copyright (2014) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level SPARTA directory. +------------------------------------------------------------------------- */ + +#include "fix_field_grid_kokkos.h" +#include "grid.h" +#include "memory_kokkos.h" +#include +#include "sparta_masks.h" + +using namespace SPARTA_NS; + +/* ---------------------------------------------------------------------- */ + +FixFieldGridKokkos::FixFieldGridKokkos(SPARTA *sparta, int narg, char **arg) : + FixFieldGrid(sparta, narg, arg) +{ + // the variable evaluation in compute_field() runs on the host, so this fix + // is a host fix. it is not invoked through Modify -- UpdateKokkos calls + // compute_field() directly -- so the datamasks only describe the fact + // that no particle data is read or written here + + kokkos_flag = 0; + execution_space = Host; + datamask_read = EMPTY_MASK; + datamask_modify = EMPTY_MASK; +} + +/* ---------------------------------------------------------------------- */ + +FixFieldGridKokkos::~FixFieldGridKokkos() +{ +} + +/* ---------------------------------------------------------------------- + evaluate the per-grid-cell field on the host, then publish it to the + device view UpdateKokkos::field_per_grid() reads +------------------------------------------------------------------------- */ + +void FixFieldGridKokkos::compute_field() +{ + FixFieldGrid::compute_field(); + + const int nglocal = grid->nlocal; + if (!nglocal) return; + const int ncols = size_per_grid_cols; + + if ((int) k_array_grid.extent(0) < nglocal || + (int) k_array_grid.extent(1) != ncols) + MemKK::realloc_kokkos(k_array_grid,"field/grid/kk:array_grid",nglocal,ncols); + + // array_grid is one contiguous row-major block: Memory::create(TYPE**&,n1,n2) + // (memory.h:114-127) does a single allocation of n1*n2 and points each + // row pointer into it. tdual_float_2d_lr is LayoutRight, so the two have + // the same element order and the host side needs no copy at all -- wrap + // the existing buffer in an unmanaged View and hand it straight to + // deep_copy. On a host backend the DualView's two views are the same + // memory and this is a no-op; on a GPU it is the one H2D transfer that + // has to happen either way. The element-wise loop this replaces was + // pure overhead on every step. + + static_assert(std::is_same::value, + "wrapping array_grid (double**) in an F_FLOAT view assumes " + "F_FLOAT is double; use a converting deep_copy if that changes"); + + Kokkos::View > + h_array_grid(array_grid[0],nglocal,ncols); + + // both sides can be longer than nglocal: the host array is sized to + // maxparticle/maxgrid, and the DualView guard above is grow-only + // (extent(0) < nglocal), so it keeps a high-water-mark row count after the + // count drops. Kokkos::deep_copy throws on any extent mismatch + // (Kokkos_CopyViews.hpp:1163), so copy the leading nglocal rows rather + // than the whole allocation. A leading row range of a LayoutRight view + // is still contiguous, so the subview costs nothing at runtime + + auto d_rows = Kokkos::subview(k_array_grid.view_device(), + Kokkos::make_pair(0,nglocal),Kokkos::ALL()); + Kokkos::deep_copy(d_rows,h_array_grid); + k_array_grid.modify_device(); + + d_array_grid = k_array_grid.view_device(); +} diff --git a/src/KOKKOS/fix_field_grid_kokkos.h b/src/KOKKOS/fix_field_grid_kokkos.h new file mode 100644 index 000000000..6d66d1b78 --- /dev/null +++ b/src/KOKKOS/fix_field_grid_kokkos.h @@ -0,0 +1,49 @@ +/* ---------------------------------------------------------------------- + SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer + http://sparta.github.io + Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov + Sandia National Laboratories + + Copyright (2014) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level SPARTA directory. +------------------------------------------------------------------------- */ + +#ifdef FIX_CLASS + +FixStyle(field/grid/kk,FixFieldGridKokkos) + +#else + +#ifndef SPARTA_FIX_FIELD_GRID_KOKKOS_H +#define SPARTA_FIX_FIELD_GRID_KOKKOS_H + +#include "fix_field_grid.h" +#include "kokkos_base.h" +#include "kokkos_type.h" + +namespace SPARTA_NS { + +// the grid-style variables this fix evaluates are host-only (see +// VariableKokkos), so the evaluation itself stays on the host and only the +// result is published to the device. that is cheap here: UpdateKokkos +// calls compute_field() once per fieldfreq steps, or once per run when +// fieldfreq is 0, not once per timestep + +class FixFieldGridKokkos : public FixFieldGrid, public KokkosBase { + public: + FixFieldGridKokkos(class SPARTA *, int, char **); + ~FixFieldGridKokkos() override; + void compute_field() override; + + private: + DAT::tdual_float_2d_lr k_array_grid; +}; + +} + +#endif +#endif diff --git a/src/KOKKOS/fix_field_particle_kokkos.cpp b/src/KOKKOS/fix_field_particle_kokkos.cpp new file mode 100644 index 000000000..68f97505b --- /dev/null +++ b/src/KOKKOS/fix_field_particle_kokkos.cpp @@ -0,0 +1,97 @@ +/* ---------------------------------------------------------------------- + SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer + http://sparta.github.io + Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov + Sandia National Laboratories + + Copyright (2014) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level SPARTA directory. +------------------------------------------------------------------------- */ + +#include "fix_field_particle_kokkos.h" +#include "particle.h" +#include "memory_kokkos.h" +#include +#include "sparta_masks.h" + +using namespace SPARTA_NS; + +/* ---------------------------------------------------------------------- */ + +FixFieldParticleKokkos::FixFieldParticleKokkos(SPARTA *sparta, int narg, char **arg) : + FixFieldParticle(sparta, narg, arg) +{ + // the variable evaluation in compute_field() runs on the host, so this fix + // is a host fix. it is not invoked through Modify -- UpdateKokkos calls + // compute_field() directly -- so the datamasks only describe the fact + // that no particle data is written here; VariableKokkos does its own + // sync(Host,PARTICLE_MASK) around the particle-style evaluation + + kokkos_flag = 0; + execution_space = Host; + datamask_read = EMPTY_MASK; + datamask_modify = EMPTY_MASK; +} + +/* ---------------------------------------------------------------------- */ + +FixFieldParticleKokkos::~FixFieldParticleKokkos() +{ +} + +/* ---------------------------------------------------------------------- + evaluate the per-particle field on the host, then publish it to the + device view UpdateKokkos::field_per_particle() reads + indexed by particle index, matching the ordering the move kernel uses +------------------------------------------------------------------------- */ + +void FixFieldParticleKokkos::compute_field() +{ + FixFieldParticle::compute_field(); + + const int nlocal = particle->nlocal; + if (!nlocal) return; + const int ncols = size_per_particle_cols; + + if ((int) k_array_particle.extent(0) < nlocal || + (int) k_array_particle.extent(1) != ncols) + MemKK::realloc_kokkos(k_array_particle,"field/particle/kk:array_particle", + nlocal,ncols); + + // array_particle is one contiguous row-major block: Memory::create(TYPE**&,n1,n2) + // (memory.h:114-127) does a single allocation of n1*n2 and points each + // row pointer into it. tdual_float_2d_lr is LayoutRight, so the two have + // the same element order and the host side needs no copy at all -- wrap + // the existing buffer in an unmanaged View and hand it straight to + // deep_copy. On a host backend the DualView's two views are the same + // memory and this is a no-op; on a GPU it is the one H2D transfer that + // has to happen either way. The element-wise loop this replaces was + // pure overhead on every step. + + static_assert(std::is_same::value, + "wrapping array_particle (double**) in an F_FLOAT view assumes " + "F_FLOAT is double; use a converting deep_copy if that changes"); + + Kokkos::View > + h_array_particle(array_particle[0],nlocal,ncols); + + // both sides can be longer than nlocal: the host array is sized to + // maxparticle/maxgrid, and the DualView guard above is grow-only + // (extent(0) < nlocal), so it keeps a high-water-mark row count after the + // count drops. Kokkos::deep_copy throws on any extent mismatch + // (Kokkos_CopyViews.hpp:1163), so copy the leading nlocal rows rather + // than the whole allocation. A leading row range of a LayoutRight view + // is still contiguous, so the subview costs nothing at runtime + + auto d_rows = Kokkos::subview(k_array_particle.view_device(), + Kokkos::make_pair(0,nlocal),Kokkos::ALL()); + Kokkos::deep_copy(d_rows,h_array_particle); + k_array_particle.modify_device(); + + d_array_particle = k_array_particle.view_device(); +} diff --git a/src/KOKKOS/fix_field_particle_kokkos.h b/src/KOKKOS/fix_field_particle_kokkos.h new file mode 100644 index 000000000..0c50b9f77 --- /dev/null +++ b/src/KOKKOS/fix_field_particle_kokkos.h @@ -0,0 +1,49 @@ +/* ---------------------------------------------------------------------- + SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer + http://sparta.github.io + Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov + Sandia National Laboratories + + Copyright (2014) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level SPARTA directory. +------------------------------------------------------------------------- */ + +#ifdef FIX_CLASS + +FixStyle(field/particle/kk,FixFieldParticleKokkos) + +#else + +#ifndef SPARTA_FIX_FIELD_PARTICLE_KOKKOS_H +#define SPARTA_FIX_FIELD_PARTICLE_KOKKOS_H + +#include "fix_field_particle.h" +#include "kokkos_base.h" +#include "kokkos_type.h" + +namespace SPARTA_NS { + +// the particle-style variables this fix evaluates are host-only (see +// VariableKokkos), so the evaluation itself stays on the host and only the +// result is published to the device. unlike field/grid this runs every +// timestep (update.cpp:672), so it carries a per-step host round trip -- +// correctness first; a device variable evaluator would remove it + +class FixFieldParticleKokkos : public FixFieldParticle, public KokkosBase { + public: + FixFieldParticleKokkos(class SPARTA *, int, char **); + ~FixFieldParticleKokkos() override; + void compute_field() override; + + private: + DAT::tdual_float_2d_lr k_array_particle; +}; + +} + +#endif +#endif diff --git a/src/KOKKOS/fix_grid_check_kokkos.cpp b/src/KOKKOS/fix_grid_check_kokkos.cpp index 2e2babb73..62514437f 100644 --- a/src/KOKKOS/fix_grid_check_kokkos.cpp +++ b/src/KOKKOS/fix_grid_check_kokkos.cpp @@ -35,8 +35,18 @@ FixGridCheckKokkos::FixGridCheckKokkos(SPARTA *sparta, int narg, char **arg) : { kokkos_flag = 1; execution_space = Device; - datamask_read = EMPTY_MASK; + datamask_read = PARTICLE_MASK; datamask_modify = EMPTY_MASK; + + // With outside set the base class also checks whether a particle in a cell + // holding surfs is inside them. That test needs grid->point_outside_surfs() + // and the cut2d/cut3d machinery behind it, none of which has a device + // implementation, so this style cannot honour the keyword. Refuse it rather + // than accept it and quietly run the smaller check: a grid checker that + // tests less than it was asked to is worse than one that is not there. + + if (outside_check) + error->all(FLERR,"Cannot (yet) use fix grid/check/kk with outside yes"); } /* ---------------------------------------------------------------------- */ @@ -45,15 +55,24 @@ void FixGridCheckKokkos::end_of_step() { if (update->ntimestep % nevery) return; + // sync through ParticleKokkos::sync()/GridKokkos::sync() rather than calling + // sync_device() on the dual views directly. The wrappers declare the host + // modified first when automatic syncing is on, which is what carries writes + // made through the plain particles/cells/cinfo pointers by non-Kokkos code + // down to the device; the bare dual-view call sees no claim, copies nothing + // and leaves the kernel below reading whatever the device happened to hold. + // This fix runs at end_of_step, right after grid adaptation and load + // balancing have rewritten those arrays on the host, so it is exactly the + // caller that cannot afford to skip it. GridKokkos::sync() also carries the + // prewrap guard. + auto particleKK = dynamic_cast(particle); - particleKK->k_particles.sync_device(); + particleKK->sync(Device,PARTICLE_MASK); auto d_particles = particleKK->k_particles.view_device(); auto gridKK = dynamic_cast(grid); - gridKK->k_cells.sync_device(); + gridKK->sync(Device,CELL_MASK|CINFO_MASK|SINFO_MASK); auto d_cells = gridKK->k_cells.view_device(); - gridKK->k_cinfo.sync_device(); auto d_cinfo = gridKK->k_cinfo.view_device(); - gridKK->k_sinfo.sync_device(); auto d_sinfo = gridKK->k_sinfo.view_device(); int nglocal = grid->nlocal; int nlocal = particle->nlocal; @@ -136,7 +155,7 @@ void FixGridCheckKokkos::end_of_step() //if (!flag) { // if (outflag == ERROR) { // char str[128]; - // sprintf(str, + // snprintf(str,sizeof(str), // "Particle %d,%d on proc %d is inside surfs in cell " // CELLINT_FORMAT " on timestep " BIGINT_FORMAT, // i,particles[i].id,comm->me,cells[icell].id, @@ -154,7 +173,7 @@ void FixGridCheckKokkos::end_of_step() // if (subcell != icell) { // if (outflag == ERROR) { // char str[128]; - // sprintf(str, + // snprintf(str,sizeof(str), // "Particle %d,%d on proc %d is in wrong sub cell %d not %d" // " on timestep " BIGINT_FORMAT, // i,particles[i].id,comm->me,icell,subcell, @@ -176,7 +195,7 @@ void FixGridCheckKokkos::end_of_step() MPI_Allreduce(&nflag,&all,1,MPI_SPARTA_BIGINT,MPI_SUM,world); if (all && comm->me == 0) { char str[128]; - sprintf(str,BIGINT_FORMAT " particles were in wrong cells on timestep " + snprintf(str,sizeof(str),BIGINT_FORMAT " particles were in wrong cells on timestep " BIGINT_FORMAT,all,update->ntimestep); error->warning(FLERR,str); } @@ -190,14 +209,14 @@ void FixGridCheckKokkos::end_of_step() for (int i = 0; i < nlocal; ++i) { auto icell = particles[i].icell; if (h_particle_problems(i) & IS_IN_INVALID_CELL) { - sprintf(str, + snprintf(str,sizeof(str), "Particle %d,%d on proc %d is in invalid cell index %d" " on timestep " BIGINT_FORMAT, i,particles[i].id,comm->me,icell,update->ntimestep); error->one(FLERR,str); } if (h_particle_problems(i) & IS_OUTSIDE_CELL) { - sprintf(str, + snprintf(str,sizeof(str), "Particle %d,%d on proc %d is outside cell " CELLINT_FORMAT " on timestep " BIGINT_FORMAT, i,particles[i].id,comm->me,cells[icell].id, @@ -205,7 +224,7 @@ void FixGridCheckKokkos::end_of_step() error->one(FLERR,str); } if (h_particle_problems(i) & IS_IN_SPLIT_CELL) { - sprintf(str, + snprintf(str,sizeof(str), "Particle %d,%d on proc %d is in split cell " CELLINT_FORMAT " on timestep " BIGINT_FORMAT, i,particles[i].id,comm->me,cells[icell].id, @@ -213,14 +232,14 @@ void FixGridCheckKokkos::end_of_step() error->one(FLERR,str); } if (h_particle_problems(i) & IS_IN_INTERIOR_CELL) { - sprintf(str, + snprintf(str,sizeof(str), "Particle %d,%d on proc %d is in interior cell " CELLINT_FORMAT " on timestep " BIGINT_FORMAT, i,particles[i].id,comm->me,cells[icell].id,update->ntimestep); error->one(FLERR,str); } if (h_particle_problems(i) & IS_IN_ZERO_VOLUME_CELL) { - sprintf(str, + snprintf(str,sizeof(str), "Particle %d,%d on proc %d is in volume=0 cell " CELLINT_FORMAT " on timestep " BIGINT_FORMAT, i,particles[i].id,comm->me,cells[icell].id,update->ntimestep); diff --git a/src/KOKKOS/fix_temp_rescale_kokkos.cpp b/src/KOKKOS/fix_temp_rescale_kokkos.cpp index 2978abcff..ca9006179 100644 --- a/src/KOKKOS/fix_temp_rescale_kokkos.cpp +++ b/src/KOKKOS/fix_temp_rescale_kokkos.cpp @@ -83,6 +83,9 @@ void FixTempRescaleKokkos::end_of_step_no_average(double t_target_in) copymode = 0; + // the kernel rescaled the thermal velocities in place on the device + particle_kk->modify(Device,PARTICLE_MASK); + d_plist = {}; } @@ -217,6 +220,10 @@ void FixTempRescaleKokkos::end_of_step_average(double t_target_in) Kokkos::parallel_for(Kokkos::RangePolicy(0,nglocal),*this); copymode = 0; + + // the kernel rescaled the thermal velocities in place on the device + particle_kk->modify(Device,PARTICLE_MASK); + d_plist = {}; } diff --git a/src/KOKKOS/grid_custom_kokkos.cpp b/src/KOKKOS/grid_custom_kokkos.cpp index 80710545d..caedc7ff1 100644 --- a/src/KOKKOS/grid_custom_kokkos.cpp +++ b/src/KOKKOS/grid_custom_kokkos.cpp @@ -165,13 +165,13 @@ void GridKokkos::allocate_custom(int index) if (esize[index] == 0) { int *ivector = eivec[ewhich[index]]; auto k_ivector = k_eivec.view_host()[ewhich[index]].k_view; - memoryKK->grow_kokkos(k_ivector,ivector,n,"surf:ivector"); + memoryKK->grow_kokkos(k_ivector,ivector,n,"grid:ivector"); k_eivec.view_host()[ewhich[index]].k_view = k_ivector; eivec[ewhich[index]] = ivector; } else { int **iarray = eiarray[ewhich[index]]; auto k_iarray = k_eiarray.view_host()[ewhich[index]].k_view; - memoryKK->grow_kokkos(k_iarray,iarray,n,esize[index],"surf:iarray"); + memoryKK->grow_kokkos(k_iarray,iarray,n,esize[index],"grid:iarray"); k_eiarray.view_host()[ewhich[index]].k_view = k_iarray; eiarray[ewhich[index]] = iarray; } @@ -180,13 +180,13 @@ void GridKokkos::allocate_custom(int index) if (esize[index] == 0) { double *dvector = edvec[ewhich[index]]; auto k_dvector = k_edvec.view_host()[ewhich[index]].k_view; - memoryKK->grow_kokkos(k_dvector,dvector,n,"surf:dvector"); + memoryKK->grow_kokkos(k_dvector,dvector,n,"grid:dvector"); k_edvec.view_host()[ewhich[index]].k_view = k_dvector; edvec[ewhich[index]] = dvector; } else { double **darray = edarray[ewhich[index]]; auto k_darray = k_edarray.view_host()[ewhich[index]].k_view; - memoryKK->grow_kokkos(k_darray,darray,n,esize[index],"surf:darray"); + memoryKK->grow_kokkos(k_darray,darray,n,esize[index],"grid:darray"); k_edarray.view_host()[ewhich[index]].k_view = k_darray; edarray[ewhich[index]] = darray; } @@ -225,13 +225,13 @@ void GridKokkos::reallocate_custom(int /*nold*/, int nnew) if (esize[ic] == 0) { int *ivector = eivec[ewhich[ic]]; auto k_ivector = k_eivec.view_host()[ewhich[ic]].k_view; - memoryKK->grow_kokkos(k_ivector,ivector,nnew,"surf:ivector"); + memoryKK->grow_kokkos(k_ivector,ivector,nnew,"grid:ivector"); k_eivec.view_host()[ewhich[ic]].k_view = k_ivector; eivec[ewhich[ic]] = ivector; } else { int **iarray = eiarray[ewhich[ic]]; auto k_iarray = k_eiarray.view_host()[ewhich[ic]].k_view; - memoryKK->grow_kokkos(k_iarray,iarray,nnew,esize[ic],"surf:iarray"); + memoryKK->grow_kokkos(k_iarray,iarray,nnew,esize[ic],"grid:iarray"); k_eiarray.view_host()[ewhich[ic]].k_view = k_iarray; eiarray[ewhich[ic]] = iarray; } @@ -240,13 +240,13 @@ void GridKokkos::reallocate_custom(int /*nold*/, int nnew) if (esize[ic] == 0) { double *dvector = edvec[ewhich[ic]]; auto k_dvector = k_edvec.view_host()[ewhich[ic]].k_view; - memoryKK->grow_kokkos(k_dvector,dvector,nnew,"surf:dvector"); + memoryKK->grow_kokkos(k_dvector,dvector,nnew,"grid:dvector"); k_edvec.view_host()[ewhich[ic]].k_view = k_dvector; edvec[ewhich[ic]] = dvector; } else { double **darray = edarray[ewhich[ic]]; auto k_darray = k_edarray.view_host()[ewhich[ic]].k_view; - memoryKK->grow_kokkos(k_darray,darray,nnew,esize[ic],"surf:darray"); + memoryKK->grow_kokkos(k_darray,darray,nnew,esize[ic],"grid:darray"); k_edarray.view_host()[ewhich[ic]].k_view = k_darray; edarray[ewhich[ic]] = darray; } diff --git a/src/KOKKOS/grid_kokkos.cpp b/src/KOKKOS/grid_kokkos.cpp index d6ef96a8c..39682e096 100644 --- a/src/KOKKOS/grid_kokkos.cpp +++ b/src/KOKKOS/grid_kokkos.cpp @@ -26,19 +26,16 @@ #include "surf_kokkos.h" #include "particle_kokkos.h" +#include + using namespace SPARTA_NS; using namespace MathConst; #define DELTA 8192 #define DELTAPARENT 1024 #define BIG 1.0e20 -#define MAXGROUP 32 #define MAXLEVEL 32 -// default value, can be overridden by global command - -#define MAXSURFPERCELL 100 - enum{XLO,XHI,YLO,YHI,ZLO,ZHI,INTERIOR}; // same as Domain enum{PERIODIC,OUTFLOW,REFLECT,SURFACE,AXISYM}; // same as Domain enum{REGION_ALL,REGION_ONE,REGION_CENTER}; // same as Surf @@ -474,3 +471,65 @@ void GridKokkos::modify(ExecutionSpace space, unsigned int mask) } } } + +/* ---------------------------------------------------------------------- + memory usage of Kokkos-managed data + Grid::memory_usage() is deliberately not called: the cells/cinfo/sinfo + arrays it measures are the host mirrors of the DualViews below. its + other two terms, the csurfs and csplits host pages, have no Kokkos + counterpart and are carried over here + the flattened Crs graphs, the per-cell particle lists and the halo index + are device-only in both backends +------------------------------------------------------------------------- */ + +bigint GridKokkos::memory_usage() +{ + const bool device_distinct = + !std::is_same::value; + + bigint bytes = csurfs->size(); + bytes += csplits->size(); + + bytes += MemKK::memory_usage(k_cells.view_host()); + bytes += MemKK::memory_usage(k_cinfo.view_host()); + bytes += MemKK::memory_usage(k_sinfo.view_host()); + bytes += MemKK::memory_usage(k_pcells.view_host()); + bytes += MemKK::memory_usage(k_plevels.view_host()); + for (int i = 0; i < ncustom_ivec; i++) + bytes += MemKK::memory_usage(k_eivec.view_host()[i].k_view.view_host()); + for (int i = 0; i < ncustom_iarray; i++) + bytes += MemKK::memory_usage(k_eiarray.view_host()[i].k_view.view_host()); + for (int i = 0; i < ncustom_dvec; i++) + bytes += MemKK::memory_usage(k_edvec.view_host()[i].k_view.view_host()); + for (int i = 0; i < ncustom_darray; i++) + bytes += MemKK::memory_usage(k_edarray.view_host()[i].k_view.view_host()); + + if (device_distinct) { + bytes += MemKK::memory_usage(k_cells.view_device()); + bytes += MemKK::memory_usage(k_cinfo.view_device()); + bytes += MemKK::memory_usage(k_sinfo.view_device()); + bytes += MemKK::memory_usage(k_pcells.view_device()); + bytes += MemKK::memory_usage(k_plevels.view_device()); + for (int i = 0; i < ncustom_ivec; i++) + bytes += MemKK::memory_usage(k_eivec.view_host()[i].k_view.view_device()); + for (int i = 0; i < ncustom_iarray; i++) + bytes += MemKK::memory_usage(k_eiarray.view_host()[i].k_view.view_device()); + for (int i = 0; i < ncustom_dvec; i++) + bytes += MemKK::memory_usage(k_edvec.view_host()[i].k_view.view_device()); + for (int i = 0; i < ncustom_darray; i++) + bytes += MemKK::memory_usage(k_edarray.view_host()[i].k_view.view_device()); + } + + bytes += MemKK::memory_usage(d_csurfs.entries); + bytes += MemKK::memory_usage(d_csurfs.row_map); + bytes += MemKK::memory_usage(d_csplits.entries); + bytes += MemKK::memory_usage(d_csplits.row_map); + bytes += MemKK::memory_usage(d_csubs.entries); + bytes += MemKK::memory_usage(d_csubs.row_map); + + bytes += MemKK::memory_usage(d_cellcount); + bytes += MemKK::memory_usage(d_plist); + bytes += MemKK::memory_usage(d_halo_index); + + return bytes; +} diff --git a/src/KOKKOS/grid_kokkos.h b/src/KOKKOS/grid_kokkos.h index 5e5273d60..f3c66c3ed 100644 --- a/src/KOKKOS/grid_kokkos.h +++ b/src/KOKKOS/grid_kokkos.h @@ -193,6 +193,7 @@ class GridKokkos : public Grid { DAT::t_int_1d d_halo_index; void update_halo_index(); + bigint memory_usage() override; DAT::tdual_int_1d k_ewhich,k_eicol,k_edcol; diff --git a/src/KOKKOS/kokkos.cpp b/src/KOKKOS/kokkos.cpp index 303d3f5db..e78dc8b4d 100644 --- a/src/KOKKOS/kokkos.cpp +++ b/src/KOKKOS/kokkos.cpp @@ -180,16 +180,32 @@ KokkosSPARTA::KokkosSPARTA(SPARTA *sparta, int narg, char **arg) : Pointers(spar if (ngpus > 0) { comm_serial = 0; - // must match the architectures that UpdateKokkos::move() dispatches the - // ATOMIC_REDUCTION = -1 (parallel_reduce) kernel for, since the counters - // are read back from the reduction result only when atomic_reduction is 0 + // SPARTA_KOKKOS_REDUCE_ARCH (kokkos_type.h) is the single definition of + // which architectures UpdateKokkos::move() dispatches the + // ATOMIC_REDUCTION = -1 (parallel_reduce) kernel for; the counters are + // read back from the reduction result only when atomic_reduction is 0 -#if defined(KOKKOS_ARCH_AMD_GFX940) || defined(KOKKOS_ARCH_AMD_GFX942) || defined(KOKKOS_ARCH_AMD_GFX942_APU) +#if SPARTA_KOKKOS_REDUCE_ARCH atomic_reduction = 0; #else atomic_reduction = 1; #endif } else { + + // on CPU the host migrate path beats the device pack/unpack kernels, so + // it stays the default. Measured on 4 MPI ranks, Serial backend, 400k + // particles in a 20^3 grid over 400 steps with 2.65% of particles + // migrating per step, comparing the Comm section of the timing + // breakdown (median of 3): + // + // comm serial comm threaded + // free molecular 0.228 s 0.246 s (+7.6%) + // with VSS collide 0.211 s 0.259 s (+22.5%) + // + // There is no host/device transfer to avoid here, so the device path + // only adds the irregular-comm plan rebuild. Users can still ask for + // it with "package kokkos comm threaded". + comm_serial = 1; atomic_reduction = 0; } diff --git a/src/KOKKOS/kokkos_base.h b/src/KOKKOS/kokkos_base.h index 74102c20d..b1a89b4e7 100644 --- a/src/KOKKOS/kokkos_base.h +++ b/src/KOKKOS/kokkos_base.h @@ -16,6 +16,7 @@ #define KOKKOS_BASE_H #include "kokkos_type.h" +#include "region_prim_kokkos.h" namespace SPARTA_NS { @@ -25,6 +26,18 @@ class KokkosBase { // Compute virtual void compute_per_grid_kokkos() {} + + // Bring this style's per-grid output up to date on the device. + // + // d_vector_grid and d_array_grid below are read by other Kokkos styles -- + // compute lambda/grid, compute dt/grid, fix ave/histo and the rest all take + // the handle straight out of the producer and launch a kernel on it. A + // compute is asked to recompute first, so its device copy is always fresh; a + // fix is not, and fix ave/grid keeps its per-grid arrays on the host while + // grid adaptation moves cells between them, leaving the device holding the + // pre-migration values. A style that can be in that position overrides this + // and the reader calls it before it reads. + virtual void sync_pergrid_device_kokkos() {} virtual int query_tally_grid_kokkos(DAT::t_float_2d_lr&) {return 0;} virtual void post_process_grid_kokkos(int, int, DAT::t_float_2d_lr, int *, DAT::t_float_1d_strided) {} @@ -41,6 +54,17 @@ class KokkosBase { // Region virtual void match_all_kokkos(DAT::tdual_int_1d) {} + // flatten this region into a device-resident postfix (RPN) token stream, + // so a kernel can test a point against it without virtual dispatch and + // without the caller holding a typed copy of every region style. + // see region_prim_kokkos.h for the encoding. + // fills the DualView (already synced to device) and returns the number + // of tokens in it. returns 0 if this region cannot be flattened. + // a composite region ignores the DualView it is handed and returns its + // own buffer, which stays valid until that same region flattens again; + // a primitive fills the DualView it is handed, growing it if needed. + virtual int flatten_region_kokkos(tdual_region_token_1d &) {return 0;} + KOKKOS_INLINE_FUNCTION int match_kokkos(double x, double y, double z) const {return 0;} }; diff --git a/src/KOKKOS/kokkos_type.h b/src/KOKKOS/kokkos_type.h index c1148aa94..de7c21d5e 100644 --- a/src/KOKKOS/kokkos_type.h +++ b/src/KOKKOS/kokkos_type.h @@ -24,6 +24,8 @@ #include "spatype.h" #include "accelerator_kokkos_defs.h" +#include + // offset type for the Kokkos::Crs per-cell surf/split/sub lists. // Under BIGBIG the total number of flattened entries on one rank can exceed // 2^31, so the row_map offsets must be a bigint. Under BIG they cannot, and @@ -37,11 +39,88 @@ typedef SPARTA_NS::bigint crs_size_type; typedef int crs_size_type; #endif -#define MAX_TYPES_STACKPARAMS 12 #define NeighClusterSize 8 +// SPARTA_KOKKOS_FIXED_LISTS restores the original fixed-size KKCopy arrays for +// the per-type tally compute lists and for the per-style surf react lists, +// in place of the runtime-sized device buffers that replaced them. The +// buffers exist to lift the instance caps below; the fixed arrays keep every +// compute and surf react model inside the functor that is handed by value to +// each kernel. +// Which is faster is a GPU question with no obvious answer: a smaller functor +// can raise occupancy while costing data locality, and higher occupancy is +// not the same thing as higher throughput. Neither path has been measured +// on an accelerator. Both are kept so the two can be compared on real +// hardware by rebuilding with -DSPARTA_KOKKOS_FIXED_LISTS, rather than by +// reverting commits. + +#ifdef SPARTA_KOKKOS_FIXED_LISTS +#define KOKKOS_MAX_SLIST 2 +#define KOKKOS_MAX_BLIST 2 +#define KOKKOS_MAX_GLIST 4 #define KOKKOS_MAX_SURF_REACT_PER_TYPE 2 #define KOKKOS_MAX_TOT_SURF_REACT 4 +#endif + +// architectures where the move kernel is dispatched with ATOMIC_REDUCTION = -1 +// (parallel_reduce) rather than atomics. KokkosSPARTA::accelerator() clears +// atomic_reduction for exactly these, and UpdateKokkos::move() reads the +// per-step counters back from the reduction result only when it is clear. +// The two decisions must agree or the counters come from the wrong place, so +// the condition is spelled out once here instead of in both files. + +#if defined(KOKKOS_ARCH_AMD_GFX940) || defined(KOKKOS_ARCH_AMD_GFX942) || \ + defined(KOKKOS_ARCH_AMD_GFX942_APU) +#define SPARTA_KOKKOS_REDUCE_ARCH 1 +#else +#define SPARTA_KOKKOS_REDUCE_ARCH 0 +#endif + +// the active surf react models, as named by the eight classes that dispatch to +// them: compute surf, and the seven surf collide models that support surface +// chemistry. Both representations are reached through these accessors, so +// the device dispatch site in each of those classes is written exactly once +// and the two modes cannot drift: +// +// KK_SR_* read on device, from the model's collide_kokkos() +// KK_SR_H_* the host image of the same models, used by the +// pre_react()/post_react()/backup()/restore() lifecycle +// +// under SPARTA_KOKKOS_FIXED_LISTS both are the same fixed KKCopy arrays, held +// by value in the class; otherwise the device side reads the per-style device +// buffer and the host side the host half of the same DualView. +// the accessors expand to member accesses only, and all eight classes spell +// those members identically, so one definition here serves every one of them. + +#ifdef SPARTA_KOKKOS_FIXED_LISTS + +#define KK_SR_TYPE(n) sr_type_list[n] +#define KK_SR_MAP(n) sr_map[n] +#define KK_SR_GLOBAL(m) sr_kk_global_copy[m].obj +#define KK_SR_PROB(m) sr_kk_prob_copy[m].obj +#define KK_SR_ADSORB(m) sr_kk_adsorb_copy[m].obj + +#define KK_SR_H_TYPE(n) sr_type_list[n] +#define KK_SR_H_MAP(n) sr_map[n] +#define KK_SR_H_GLOBAL(m) sr_kk_global_copy[m].obj +#define KK_SR_H_PROB(m) sr_kk_prob_copy[m].obj +#define KK_SR_H_ADSORB(m) sr_kk_adsorb_copy[m].obj + +#else + +#define KK_SR_TYPE(n) d_sr_type_list[n] +#define KK_SR_MAP(n) d_sr_map[n] +#define KK_SR_GLOBAL(m) ((const SurfReactGlobalKokkos *) d_sr_global.data())[m] +#define KK_SR_PROB(m) ((const SurfReactProbKokkos *) d_sr_prob.data())[m] +#define KK_SR_ADSORB(m) ((const SurfReactAdsorbKokkos *) d_sr_adsorb.data())[m] + +#define KK_SR_H_TYPE(n) k_sr_type_list.view_host()[n] +#define KK_SR_H_MAP(n) k_sr_map.view_host()[n] +#define KK_SR_H_GLOBAL(m) ((SurfReactGlobalKokkos *) k_sr_global.view_host().data())[m] +#define KK_SR_H_PROB(m) ((SurfReactProbKokkos *) k_sr_prob.view_host().data())[m] +#define KK_SR_H_ADSORB(m) ((SurfReactAdsorbKokkos *) k_sr_adsorb.view_host().data())[m] + +#endif namespace Kokkos { static auto NoInit = [](std::string const& label) { @@ -746,6 +825,68 @@ namespace SPARTA_NS { typedef Kokkos::DualView tdual_struct_tdual_float_2d_1d; } +#ifndef SPARTA_KOKKOS_FIXED_LISTS + +// the per-style device buffers behind the KK_SR_* accessors above, and the two +// index lists that map a surf react index to a style and to a slot within it. +// blitting a model into a buffer is the same operation, for the same reason, as +// KKCopy::copy() (kokkos_copy.h:71): on device the model is only read, through +// KOKKOS_INLINE_FUNCTION members, so its vtable pointer is never used and the +// Views it carries stay alive in the original that surf->sr holds. The host +// lifecycle calls are made on the host half of the same bytes -- a valid +// object of the class as far as the host is concerned, its vtable pointer +// copied from a live instance -- and pushed to the device by sr_buf_sync(). +// shared here rather than repeated in each of the eight classes that carry +// these lists, since all eight set them up the same way. + +namespace SPARTA_NS { + + template + void sr_buf_resize(DAT::tdual_char_1d &k, DAT::t_char_1d &d, int n) + { + const size_t need = (size_t) (n > 0 ? n : 1) * sizeof(T); + if (k.view_device().extent(0) < need) { + k = DAT::tdual_char_1d("surf_react:models",need); + d = k.view_device(); + } + } + + template + void sr_buf_blit(DAT::tdual_char_1d &k, int slot, T *obj) + { + char *dst = k.view_host().data() + (size_t) slot*sizeof(T); + memcpy((void*) dst, (const void*) obj, sizeof(T)); + ((T *) dst)->copy = 1; + } + + inline void sr_buf_sync(DAT::tdual_char_1d &k, DAT::t_char_1d &d) + { + if (k.view_device().extent(0) == 0) return; + k.modify_host(); + k.sync_device(); + d = k.view_device(); + } + + inline void sr_idx_resize(DAT::tdual_int_1d &k, DAT::t_int_1d &d, int n) + { + const size_t need = (size_t) (n > 0 ? n : 1); + if (k.view_device().extent(0) < need) { + k = DAT::tdual_int_1d("surf_react:index",need); + d = k.view_device(); + } + } + + inline void sr_idx_sync(DAT::tdual_int_1d &k, DAT::t_int_1d &d) + { + if (k.view_device().extent(0) == 0) return; + k.modify_host(); + k.sync_device(); + d = k.view_device(); + } +} + +#endif + template void buffer_view(BufferView &buf, DualView &view, const size_t n0, diff --git a/src/KOKKOS/modify_kokkos.cpp b/src/KOKKOS/modify_kokkos.cpp index 19a1504d6..3c57eeece 100644 --- a/src/KOKKOS/modify_kokkos.cpp +++ b/src/KOKKOS/modify_kokkos.cpp @@ -288,17 +288,23 @@ void ModifyKokkos::gas_react(int index) invoke surf_react() method, only for relevant fixes ------------------------------------------------------------------------- */ -void ModifyKokkos::surf_react(Particle::OnePart *iorig, int &i, int &) +void ModifyKokkos::surf_react(Particle::OnePart *iorig, int &i, int &j) { + // the fix index is ifix here, not j: j is the caller's index of the second + // post-reaction particle (negative for an exchange reaction), which the + // fix reads and writes back -- e.g. FixAmbipolar::surf_react() sets it to + // -1 to tell the caller to delete an electron it just created. Shadowing + // it with the fix index would hand the fix a particle index it never meant + for (int m = 0; m < n_surf_react; m++) { - int j = list_surf_react[m]; - particle_kk->sync(fix[j]->execution_space,fix[j]->datamask_read); + int ifix = list_surf_react[m]; + particle_kk->sync(fix[ifix]->execution_space,fix[ifix]->datamask_read); int prev_auto_sync = sparta->kokkos->auto_sync; - if (!fix[j]->kokkos_flag) sparta->kokkos->auto_sync = 1; + if (!fix[ifix]->kokkos_flag) sparta->kokkos->auto_sync = 1; - fix[list_surf_react[m]]->surf_react(iorig,i,j); + fix[ifix]->surf_react(iorig,i,j); sparta->kokkos->auto_sync = prev_auto_sync; - particle_kk->modify(fix[j]->execution_space,fix[j]->datamask_modify); + particle_kk->modify(fix[ifix]->execution_space,fix[ifix]->datamask_modify); } } diff --git a/src/KOKKOS/particle_custom_kokkos.cpp b/src/KOKKOS/particle_custom_kokkos.cpp index 879d6c9b3..409350dab 100644 --- a/src/KOKKOS/particle_custom_kokkos.cpp +++ b/src/KOKKOS/particle_custom_kokkos.cpp @@ -367,6 +367,24 @@ void ParticleKokkos::zero_custom_kokkos() zero_custom_kokkos(nlocal,maxlocal); } +/* ---------------------------------------------------------------------- + zero the custom attributes of particle I + Particle::add_particle() calls this for every particle a host caller + creates, writing through the raw eivec/edvec/... pointers. Register + that write, or a later sync(Device,CUSTOM_MASK) is a no-op and the + device keeps whatever the slot last held. Not covered by the caller + in every case: SurfReactAdsorbKokkos inserts PS-chemistry particles on + the host and marks only PARTICLE_MASK + this is the host-side counterpart of zero_custom_kokkos() +------------------------------------------------------------------------- */ + +void ParticleKokkos::zero_custom(int i) +{ + sync(Host,CUSTOM_MASK); + Particle::zero_custom(i); + modify(Host,CUSTOM_MASK); +} + /* ---------------------------------------------------------------------- copy info for one particle in custom attribute vectors/arrays into location I from location J diff --git a/src/KOKKOS/particle_kokkos.cpp b/src/KOKKOS/particle_kokkos.cpp index 8250ad9b2..f9bf556ee 100644 --- a/src/KOKKOS/particle_kokkos.cpp +++ b/src/KOKKOS/particle_kokkos.cpp @@ -116,6 +116,19 @@ static int cellcount_target(int need, int nlocal_in, int ngrid_in, ParticleKokkos::ParticleKokkos(SPARTA *sparta) : Particle(sparta) { + // NOTE: the weight_rand_pool seed cannot be set here. Every other Kokkos + // class seeds its pool in the constructor initializer list with + // 12345 + comm->me, but those are all styles the input script creates, + // long after SPARTA::create() has finished. ParticleKokkos is built by + // create() itself, at sparta.cpp:484, three lines BEFORE comm exists + // (:487), and comm is not NULL-initialized -- so reading comm->me there + // dereferences an uninitialized pointer. Seed on first use instead. + +#ifndef SPARTA_KOKKOS_EXACT + weight_rand_pool_seeded = 0; +#endif + + d_resize = DAT::t_int_scalar("particle:resize"); h_resize = HAT::t_int_scalar("particle:resize_mirror"); @@ -262,13 +275,11 @@ void ParticleKokkos::sort_kokkos() // reorder_scheme = FIXEDMEMORY; // reordering is a memory-locality optimization only, so it is safe to skip - // skip it if custom per-particle data exists: the reorder kernels below - // permute only the OnePart records, they do not permute the custom - // vectors/arrays, which would silently decouple custom values from their - // particles. Particle::reorder() handles this on the host via - // copy_custom(); post_weight() takes the same "bail out if ncustom" tack. + // custom per-particle data is permuted alongside the OnePart records below, + // through the same d_sorted_id map, which is what Particle::reorder() + // does on the host via copy_custom() - const int reorder_flag = (update->reorder_period && !ncustom && + const int reorder_flag = (update->reorder_period && (update->ntimestep % update->reorder_period == 0)); ngrid = grid->nlocal; @@ -392,6 +403,62 @@ void ParticleKokkos::sort_kokkos() //d_sorted = tmp; Kokkos::deep_copy(d_particles,d_sorted); + // permute the custom attributes through the same d_sorted_id map, or + // the reorder would silently decouple custom values from their + // particles. gather through a temporary: not an in-place permutation + + if (ncustom) { + this->sync(Device,CUSTOM_MASK); + auto l_sorted_id = d_sorted_id; + const int l_nlocal = nlocal; + + for (int m = 0; m < ncustom_ivec; m++) { + auto d_src = k_eivec.view_host()[m].k_view.view_device(); + DAT::t_int_1d d_tmp(Kokkos::view_alloc("reorder:custom_ivec", + Kokkos::WithoutInitializing),l_nlocal); + Kokkos::parallel_for(l_nlocal, KOKKOS_LAMBDA(int i) { + d_tmp[i] = d_src[l_sorted_id[i]]; + }); + Kokkos::deep_copy(Kokkos::subview(d_src,Kokkos::make_pair(0,l_nlocal)),d_tmp); + } + + for (int m = 0; m < ncustom_iarray; m++) { + auto d_src = k_eiarray.view_host()[m].k_view.view_device(); + const int ncol = d_src.extent(1); + DAT::t_int_2d d_tmp(Kokkos::view_alloc("reorder:custom_iarray", + Kokkos::WithoutInitializing),l_nlocal,ncol); + Kokkos::parallel_for(l_nlocal, KOKKOS_LAMBDA(int i) { + for (int k = 0; k < ncol; k++) d_tmp(i,k) = d_src(l_sorted_id[i],k); + }); + Kokkos::deep_copy(Kokkos::subview(d_src,Kokkos::make_pair(0,l_nlocal), + Kokkos::ALL()),d_tmp); + } + + for (int m = 0; m < ncustom_dvec; m++) { + auto d_src = k_edvec.view_host()[m].k_view.view_device(); + DAT::t_float_1d d_tmp(Kokkos::view_alloc("reorder:custom_dvec", + Kokkos::WithoutInitializing),l_nlocal); + Kokkos::parallel_for(l_nlocal, KOKKOS_LAMBDA(int i) { + d_tmp[i] = d_src[l_sorted_id[i]]; + }); + Kokkos::deep_copy(Kokkos::subview(d_src,Kokkos::make_pair(0,l_nlocal)),d_tmp); + } + + for (int m = 0; m < ncustom_darray; m++) { + auto d_src = k_edarray.view_host()[m].k_view.view_device(); + const int ncol = d_src.extent(1); + DAT::t_float_2d d_tmp(Kokkos::view_alloc("reorder:custom_darray", + Kokkos::WithoutInitializing),l_nlocal,ncol); + Kokkos::parallel_for(l_nlocal, KOKKOS_LAMBDA(int i) { + for (int k = 0; k < ncol; k++) d_tmp(i,k) = d_src(l_sorted_id[i],k); + }); + Kokkos::deep_copy(Kokkos::subview(d_src,Kokkos::make_pair(0,l_nlocal), + Kokkos::ALL()),d_tmp); + } + + this->modify(Device,CUSTOM_MASK); + } + this->modify(Device,PARTICLE_MASK); } else if (reorder_scheme == FIXEDMEMORY) { @@ -586,12 +653,27 @@ void ParticleKokkos::pre_weight() only grid-based weighting is currently implemented ------------------------------------------------------------------------- */ -struct PostWeightPair { int i; int id; }; - void ParticleKokkos::post_weight() { + // METHOD 1 is the host fallback. it used to be taken whenever any custom + // per-particle attribute existed -- which is any run with fix ambipolar + // or fix vibmode -- costing a full particle+custom round trip on every + // timestep that grid weighting is active. METHOD 2 now permutes the + // custom arrays with the same map it uses for the particles, so the + // fallback is only kept as a reference implementation + +#ifndef SPARTA_KOKKOS_EXACT + // the loop in METHOD 2 below is serial on the host because its + // delete-by-swap-from-the-end makes the RNG draw order matter, which + // SPARTA_KOKKOS_EXACT needs in order to reproduce Particle::post_weight() + // bit-for-bit. the physics it implements is per-particle independent -- + // survive with probability ratio, or replicate to 1+nclone copies -- so + // away from EXACT it is a prefix-sum scatter with no host round trip + post_weight_device(); + return; +#endif + int METHOD = 2; - if (particle->ncustom) METHOD = 1; if (METHOD == 1) { // just call the host one this->sync(Host,PARTICLE_MASK|CUSTOM_MASK); @@ -607,11 +689,6 @@ void ParticleKokkos::post_weight() this->modify(Host,PARTICLE_MASK|CUSTOM_MASK); } else if (METHOD == 2) { // Kokkos-parallel, gives same (correct) answer - DAT::tdual_float_1d k_ratios; - MemKK::realloc_kokkos(k_ratios,"post_weight:ratios",nlocal); - auto d_ratios = k_ratios.view_device(); - auto h_ratios = k_ratios.view_host(); - auto grid_kk = dynamic_cast(grid); auto& k_cinfo = grid_kk->k_cinfo; grid_kk->sync(Device,CINFO_MASK); @@ -620,21 +697,35 @@ void ParticleKokkos::post_weight() auto d_particles = k_particles.view_device(); auto d_cinfo = k_cinfo.view_device(); - typedef Kokkos::DualView tdual_pwp_1d; - tdual_pwp_1d k_map; - MemKK::realloc_kokkos(k_map,"post_weight:map",nlocal*1.5); + // k_map persists across calls and only ever grows: this runs every + // timestep that weighting is active, and reallocating an O(nlocal) + // DualView per step is pure overhead on a GPU + + if ((int) k_map.extent(0) < nlocal) + MemKK::realloc_kokkos(k_map,"post_weight:map",(size_t)(nlocal*1.5)); auto d_map = k_map.view_device(); auto h_map = k_map.view_host(); - Kokkos::parallel_for(nlocal, KOKKOS_LAMBDA(int i) { - auto icell = d_particles[i].icell; - d_ratios[i] = d_particles[i].weight / d_cinfo[icell].weight; + // count how many particles actually changed weight while filling the map + // if none did, the host loop below would only walk the map without + // touching it and draw no random numbers, and the final gather would be + // an identity permutation -- so skip the whole round trip. this is + // exactly equivalent, RNG stream included + + int nchanged = 0; + Kokkos::parallel_reduce(nlocal, KOKKOS_LAMBDA(const int i, int &lsum) { + const auto icell = d_particles[i].icell; + const double ratio = d_particles[i].weight / d_cinfo[icell].weight; + d_map[i].ratio = ratio; d_map[i].id = d_particles[i].id; d_map[i].i = i; - }); + if (ratio != 1.0) lsum++; + },nchanged); - k_ratios.modify_device(); - k_ratios.sync_host(); + if (!nchanged) { + d_particles = t_particle_1d(); + return; + } k_map.modify_device(); k_map.sync_host(); @@ -646,7 +737,7 @@ void ParticleKokkos::post_weight() while (i < nlocal_original) { - auto ratio = h_ratios[h_map[i].i]; + auto ratio = h_map[i].ratio; // next particle will be an original particle // skip it if no weight change @@ -703,17 +794,80 @@ void ParticleKokkos::post_weight() k_map.sync_device(); grow(0); - t_particle_1d d_newparticles; - MemKK::realloc_kokkos(d_newparticles,"post_weight:newparticles", maxlocal); + + // likewise persistent: this is a full maxlocal-sized particle array, and + // allocating and freeing it every timestep is the single largest + // avoidable cost in this routine on a GPU + + if ((int) d_newparticles.extent(0) < maxlocal) + MemKK::realloc_kokkos(d_newparticles,"post_weight:newparticles",maxlocal); + auto d_newparticles_l = d_newparticles; d_map = k_map.view_device(); Kokkos::parallel_for(nlocal, KOKKOS_LAMBDA(int i) { - d_newparticles[i] = d_particles[d_map[i].i]; - d_newparticles[i].id = d_map[i].id; + d_newparticles_l[i] = d_particles[d_map[i].i]; + d_newparticles_l[i].id = d_map[i].id; }); - Kokkos::deep_copy(k_particles.view_device(),d_newparticles); + Kokkos::deep_copy(k_particles.view_device(), + Kokkos::subview(d_newparticles,Kokkos::make_pair(0,(int)k_particles.view_device().extent(0)))); this->modify(Device,PARTICLE_MASK); + + // permute the custom attributes with the same map + // a cloned particle carries its source's index in d_map, so it inherits + // that particle's custom values, matching Particle::post_weight() + // gather through a temporary: the permutation is not in place + + if (ncustom) { + this->sync(Device,CUSTOM_MASK); + + for (int m = 0; m < ncustom_ivec; m++) { + auto d_src = k_eivec.view_host()[m].k_view.view_device(); + DAT::t_int_1d d_tmp(Kokkos::view_alloc("post_weight:custom_ivec", + Kokkos::WithoutInitializing),nlocal); + Kokkos::parallel_for(nlocal, KOKKOS_LAMBDA(int i) { + d_tmp[i] = d_src[d_map[i].i]; + }); + Kokkos::deep_copy(Kokkos::subview(d_src,Kokkos::make_pair(0,nlocal)),d_tmp); + } + + for (int m = 0; m < ncustom_iarray; m++) { + auto d_src = k_eiarray.view_host()[m].k_view.view_device(); + const int ncol = d_src.extent(1); + DAT::t_int_2d d_tmp(Kokkos::view_alloc("post_weight:custom_iarray", + Kokkos::WithoutInitializing),nlocal,ncol); + Kokkos::parallel_for(nlocal, KOKKOS_LAMBDA(int i) { + for (int k = 0; k < ncol; k++) d_tmp(i,k) = d_src(d_map[i].i,k); + }); + Kokkos::deep_copy(Kokkos::subview(d_src,Kokkos::make_pair(0,nlocal), + Kokkos::ALL()),d_tmp); + } + + for (int m = 0; m < ncustom_dvec; m++) { + auto d_src = k_edvec.view_host()[m].k_view.view_device(); + DAT::t_float_1d d_tmp(Kokkos::view_alloc("post_weight:custom_dvec", + Kokkos::WithoutInitializing),nlocal); + Kokkos::parallel_for(nlocal, KOKKOS_LAMBDA(int i) { + d_tmp[i] = d_src[d_map[i].i]; + }); + Kokkos::deep_copy(Kokkos::subview(d_src,Kokkos::make_pair(0,nlocal)),d_tmp); + } + + for (int m = 0; m < ncustom_darray; m++) { + auto d_src = k_edarray.view_host()[m].k_view.view_device(); + const int ncol = d_src.extent(1); + DAT::t_float_2d d_tmp(Kokkos::view_alloc("post_weight:custom_darray", + Kokkos::WithoutInitializing),nlocal,ncol); + Kokkos::parallel_for(nlocal, KOKKOS_LAMBDA(int i) { + for (int k = 0; k < ncol; k++) d_tmp(i,k) = d_src(d_map[i].i,k); + }); + Kokkos::deep_copy(Kokkos::subview(d_src,Kokkos::make_pair(0,nlocal), + Kokkos::ALL()),d_tmp); + } + + this->modify(Device,CUSTOM_MASK); + } + d_particles = t_particle_1d(); } } @@ -927,3 +1081,213 @@ void ParticleKokkos::modify(ExecutionSpace space, unsigned int mask) } } } + +/* ---------------------------------------------------------------------- + memory usage of Kokkos-managed data + Particle::memory_usage() is deliberately not called: the host arrays it + measures are the host mirrors of the DualViews below, so its formula + would double count them. next[] is the one plain host allocation it + covers with no Kokkos counterpart, so it is carried over here + the device half is added only when it is a distinct allocation; on a + host-only backend the two views alias +------------------------------------------------------------------------- */ + +bigint ParticleKokkos::memory_usage() +{ + const bool device_distinct = + !std::is_same::value; + + bigint bytes = (bigint) maxlocal * sizeof(int); // next[] + + bytes += MemKK::memory_usage(k_particles.view_host()); + bytes += MemKK::memory_usage(k_species.view_host()); + bytes += MemKK::memory_usage(k_species2group.view_host()); + for (int i = 0; i < ncustom_ivec; i++) + bytes += MemKK::memory_usage(k_eivec.view_host()[i].k_view.view_host()); + for (int i = 0; i < ncustom_iarray; i++) + bytes += MemKK::memory_usage(k_eiarray.view_host()[i].k_view.view_host()); + for (int i = 0; i < ncustom_dvec; i++) + bytes += MemKK::memory_usage(k_edvec.view_host()[i].k_view.view_host()); + for (int i = 0; i < ncustom_darray; i++) + bytes += MemKK::memory_usage(k_edarray.view_host()[i].k_view.view_host()); + + if (device_distinct) { + bytes += MemKK::memory_usage(k_particles.view_device()); + bytes += MemKK::memory_usage(k_species.view_device()); + bytes += MemKK::memory_usage(k_species2group.view_device()); + for (int i = 0; i < ncustom_ivec; i++) + bytes += MemKK::memory_usage(k_eivec.view_host()[i].k_view.view_device()); + for (int i = 0; i < ncustom_iarray; i++) + bytes += MemKK::memory_usage(k_eiarray.view_host()[i].k_view.view_device()); + for (int i = 0; i < ncustom_dvec; i++) + bytes += MemKK::memory_usage(k_edvec.view_host()[i].k_view.view_device()); + for (int i = 0; i < ncustom_darray; i++) + bytes += MemKK::memory_usage(k_edarray.view_host()[i].k_view.view_device()); + } + + // device-only scratch for the sort/reorder path, with no host counterpart + // in either backend + + bytes += MemKK::memory_usage(d_sorted); + bytes += MemKK::memory_usage(d_sorted_id); + bytes += MemKK::memory_usage(d_offsets_part); + + return bytes; +} + +#ifndef SPARTA_KOKKOS_EXACT + +/* ---------------------------------------------------------------------- + fully on-device post_weight() + same physics as Particle::post_weight(): a particle whose weight ratio is + below 1 survives with probability ratio, and one above 1 is replicated to + 1+nclone copies, nclone drawn from the fractional part. the host version + realizes that by walking a map and swapping deleted entries in from the + end, which serializes it; here each particle decides independently, an + exclusive scan turns the per-particle copy counts into output offsets, + and one scatter writes the new list + not bit-compatible with the host RNG stream, which is why EXACT builds keep + the serial path +------------------------------------------------------------------------- */ + +void ParticleKokkos::post_weight_device() +{ + if (!nlocal) return; + + auto grid_kk = dynamic_cast(grid); + grid_kk->sync(Device,CINFO_MASK); + this->sync(Device,PARTICLE_MASK|CUSTOM_MASK); + + auto d_particles_l = k_particles.view_device(); + auto d_cinfo = grid_kk->k_cinfo.view_device(); + + // seed on first use: comm does not exist yet when this class is constructed + // (see the constructor). the seed matches every other Kokkos style's + + if (!weight_rand_pool_seeded) { + weight_rand_pool = + Kokkos::Random_XorShift64_Pool(12345 + comm->me); + weight_rand_pool_seeded = 1; + } + + auto l_pool = weight_rand_pool; + const int nold = nlocal; + + // per-particle output count + + // plain Kokkos::View, not DAT::t_int_1d: offset_scan() takes + // Kokkos::View and the DAT alias does not deduce against it + // (see the same note on FixEmitFaceKokkos::d_keep) + + Kokkos::View d_count("post_weight:count",nold); + + Kokkos::parallel_for(nold, KOKKOS_LAMBDA(const int i) { + const int icell = d_particles_l[i].icell; + const double ratio = d_particles_l[i].weight / d_cinfo[icell].weight; + + if (ratio == 1.0) { d_count[i] = 1; return; } + + rand_type rand_gen = l_pool.get_state(); + if (ratio < 1.0) { + d_count[i] = (rand_gen.drand() > ratio) ? 0 : 1; + } else { + int nclone = static_cast(ratio); + const double fraction = ratio - nclone; + nclone--; + if (rand_gen.drand() < fraction) nclone++; + d_count[i] = 1 + nclone; + } + l_pool.free_state(rand_gen); + }); + + // exclusive scan -> output offset of each particle's first copy. + // offset_scan() accumulates in 64-bit and aborts with a clear message if + // the total exceeds 2^31. a hand-rolled scan carrying an int update + // would wrap to a negative nnew first, and the overflow guard that used + // to sit here could never fire: MAXSMALLINT is INT_MAX, so "nnew > + // MAXSMALLINT" is false for every int + + int nnew = 0; + auto d_offset = offset_scan(d_count, nnew); + + // grow to the new count, then scatter + + const int nlocal_save = nlocal; + nlocal = nnew; + if (nnew > maxlocal) { + nlocal = nlocal_save; + grow(nnew - nlocal_save); + nlocal = nnew; + } + + if ((int) d_newparticles.extent(0) < maxlocal) + MemKK::realloc_kokkos(d_newparticles,"post_weight:newparticles",maxlocal); + + auto d_new = d_newparticles; + d_particles_l = k_particles.view_device(); + + Kokkos::parallel_for(nold, KOKKOS_LAMBDA(const int i) { + const int n = d_count[i]; + if (!n) return; + const int base = d_offset[i]; + d_new[base] = d_particles_l[i]; + if (n > 1) { + rand_type rand_gen = l_pool.get_state(); + for (int k = 1; k < n; k++) { + d_new[base+k] = d_particles_l[i]; + d_new[base+k].id = MAXSMALLINT*rand_gen.drand(); + } + l_pool.free_state(rand_gen); + } + }); + + Kokkos::deep_copy(Kokkos::subview(k_particles.view_device(),Kokkos::make_pair(0,nnew)), + Kokkos::subview(d_new,Kokkos::make_pair(0,nnew))); + this->modify(Device,PARTICLE_MASK); + + // permute the custom attributes through the same offsets + + if (ncustom) { + for (int m = 0; m < ncustom_ivec; m++) { + auto d_src = k_eivec.view_host()[m].k_view.view_device(); + DAT::t_int_1d d_tmp("post_weight:cust_iv",nnew); + Kokkos::parallel_for(nold, KOKKOS_LAMBDA(const int i) { + for (int k = 0; k < d_count[i]; k++) d_tmp[d_offset[i]+k] = d_src[i]; + }); + Kokkos::deep_copy(Kokkos::subview(d_src,Kokkos::make_pair(0,nnew)),d_tmp); + } + for (int m = 0; m < ncustom_iarray; m++) { + auto d_src = k_eiarray.view_host()[m].k_view.view_device(); + const int ncol = d_src.extent(1); + DAT::t_int_2d d_tmp("post_weight:cust_ia",nnew,ncol); + Kokkos::parallel_for(nold, KOKKOS_LAMBDA(const int i) { + for (int k = 0; k < d_count[i]; k++) + for (int c = 0; c < ncol; c++) d_tmp(d_offset[i]+k,c) = d_src(i,c); + }); + Kokkos::deep_copy(Kokkos::subview(d_src,Kokkos::make_pair(0,nnew),Kokkos::ALL()),d_tmp); + } + for (int m = 0; m < ncustom_dvec; m++) { + auto d_src = k_edvec.view_host()[m].k_view.view_device(); + DAT::t_float_1d d_tmp("post_weight:cust_dv",nnew); + Kokkos::parallel_for(nold, KOKKOS_LAMBDA(const int i) { + for (int k = 0; k < d_count[i]; k++) d_tmp[d_offset[i]+k] = d_src[i]; + }); + Kokkos::deep_copy(Kokkos::subview(d_src,Kokkos::make_pair(0,nnew)),d_tmp); + } + for (int m = 0; m < ncustom_darray; m++) { + auto d_src = k_edarray.view_host()[m].k_view.view_device(); + const int ncol = d_src.extent(1); + DAT::t_float_2d d_tmp("post_weight:cust_da",nnew,ncol); + Kokkos::parallel_for(nold, KOKKOS_LAMBDA(const int i) { + for (int k = 0; k < d_count[i]; k++) + for (int c = 0; c < ncol; c++) d_tmp(d_offset[i]+k,c) = d_src(i,c); + }); + Kokkos::deep_copy(Kokkos::subview(d_src,Kokkos::make_pair(0,nnew),Kokkos::ALL()),d_tmp); + } + this->modify(Device,CUSTOM_MASK); + } + + sorted_kk = 0; +} + +#endif diff --git a/src/KOKKOS/particle_kokkos.h b/src/KOKKOS/particle_kokkos.h index 359bc6d06..e1e2a4ab1 100644 --- a/src/KOKKOS/particle_kokkos.h +++ b/src/KOKKOS/particle_kokkos.h @@ -36,6 +36,12 @@ template struct TagParticleSort{}; +// map entry used by post_weight(): source particle index, the new particle +// id, and the weight ratio that decides clone/delete. ratio rides along +// in the same struct so only one array crosses the bus per timestep + +struct PostWeightPair { int i; int id; double ratio; }; + class ParticleKokkos : public Particle { public: typedef int value_type; @@ -60,6 +66,8 @@ class ParticleKokkos : public Particle { int add_custom(char *, int, int) override; void grow_custom(int, int, int) override; void remove_custom(int) override; + void zero_custom(int) override; + bigint memory_usage() override; void copy_custom(int, int) override; void pack_custom(int, char *) override; void unpack_custom(char *, int) override; @@ -79,6 +87,14 @@ class ParticleKokkos : public Particle { void zero_custom_kokkos(); #ifndef SPARTA_KOKKOS_EXACT + // pool for post_weight_device(). only the EXACT path needs to match the + // host RNG stream, so this exists only off EXACT. unlike every other + // Kokkos class it cannot be seeded in the ctor initializer list -- see + // ParticleKokkos::ParticleKokkos() -- so it is seeded on first use + Kokkos::Random_XorShift64_Pool weight_rand_pool; + int weight_rand_pool_seeded; + void post_weight_device(); + typedef typename Kokkos::Random_XorShift64_Pool::generator_type rand_type; //typedef typename Kokkos::Random_XorShift1024_Pool::generator_type rand_type; @@ -185,6 +201,10 @@ class ParticleKokkos : public Particle { // work memory for reduced memory reordering t_particle_1d d_pswap1; t_particle_1d d_pswap2; + + // persistent scratch for post_weight(); grown, never reallocated per step + Kokkos::DualView k_map; + t_particle_1d d_newparticles; }; KOKKOS_INLINE_FUNCTION diff --git a/src/KOKKOS/react_bird_kokkos.cpp b/src/KOKKOS/react_bird_kokkos.cpp index 2a46dc971..aa80af8c3 100644 --- a/src/KOKKOS/react_bird_kokkos.cpp +++ b/src/KOKKOS/react_bird_kokkos.cpp @@ -37,7 +37,6 @@ using namespace MathConst; enum{DISSOCIATION,EXCHANGE,IONIZATION,RECOMBINATION}; // other react files enum{ARRHENIUS,QUANTUM}; // other react files -#define MAXLINE 1024 #define DELTALIST 16 /* ---------------------------------------------------------------------- */ diff --git a/src/KOKKOS/react_qk_kokkos.cpp b/src/KOKKOS/react_qk_kokkos.cpp index 1e3b276eb..4088c13ac 100644 --- a/src/KOKKOS/react_qk_kokkos.cpp +++ b/src/KOKKOS/react_qk_kokkos.cpp @@ -36,7 +36,11 @@ void ReactQKKokkos::init() ReactBirdKokkos::init(); - // do not allow recombination reactions (not supported by QK) + // these two restrictions mirror the host (react_qk.cpp:46-54) exactly -- + // same condition, same message. They are NOT Kokkos limitations: + // react qk does not support recombination or compute_chem_rates on + // the CPU either, so lifting them here would diverge from the host. + // Only react tce supports both, and react_tce_kokkos.h ports that. for (int i = 0; i < nlist; i++) if (rlist[i].active && rlist[i].type == RECOMBINATION) diff --git a/src/KOKKOS/react_qk_kokkos.h b/src/KOKKOS/react_qk_kokkos.h index e2e592d72..0fed96103 100644 --- a/src/KOKKOS/react_qk_kokkos.h +++ b/src/KOKKOS/react_qk_kokkos.h @@ -98,11 +98,17 @@ int attempt_kk(Particle::OnePart *ip, Particle::OnePart *jp, ecc = pre_etrans + ip->evib; maxlev = static_cast (ecc * inverse_kT); if (ecc > r->d_coeff[1]) { + // sample into a local prob, not react_prob: react_prob is the + // reaction probability this function returns on, and a rejected + // draw must not leave a fractional value in it (or carry one + // into the next reaction). Mirrors ReactQK::attempt() + double prob = 0.0; do { iv = static_cast (rand_gen.drand()*(maxlev+0.99999999)); double evib = static_cast (iv / inverse_kT); - if (evib < ecc) react_prob = pow(1.0-evib/ecc,1.5-omega); - } while (rand_gen.drand() < react_prob); + if (evib < ecc) prob = pow(1.0-evib/ecc,1.5-omega); + else prob = 0.0; + } while (rand_gen.drand() < prob); ilevel = static_cast (fabs(r->d_coeff[4]) * inverse_kT); if (iv >= ilevel) react_prob = 1.0; @@ -124,6 +130,7 @@ int attempt_kk(Particle::OnePart *ip, Particle::OnePart *jp, iv = rand_gen.drand()*(maxlev+0.99999999); double evib = static_cast (iv * boltz*d_species[mspec].vibtemp[0]); if (evib < ecc) prob = pow(1.0-evib/ecc,1.5 - r->d_coeff[6]); + else prob = 0.0; } while (rand_gen.drand() < prob); ilevel = static_cast (fabs(r->d_coeff[4]/boltz/d_species[mspec].vibtemp[0])); diff --git a/src/KOKKOS/react_tce_kokkos.h b/src/KOKKOS/react_tce_kokkos.h index c54181de3..c57bc7e0d 100644 --- a/src/KOKKOS/react_tce_kokkos.h +++ b/src/KOKKOS/react_tce_kokkos.h @@ -323,6 +323,13 @@ int attempt_kk(Particle::OnePart *ip, Particle::OnePart *jp, return d_list[i] + 1; } + + // computeChemRates: the tally above is the whole point of this pass and + // no reaction is performed, but the search still stops at the first + // reaction that passes, as ReactTCE::attempt() does. Without this the + // loop keeps going and tallies every later reaction that also passes + + break; } } diff --git a/src/KOKKOS/react_tce_qk_kokkos.cpp b/src/KOKKOS/react_tce_qk_kokkos.cpp index 0cc06574b..c809cf86b 100644 --- a/src/KOKKOS/react_tce_qk_kokkos.cpp +++ b/src/KOKKOS/react_tce_qk_kokkos.cpp @@ -36,7 +36,11 @@ void ReactTCEQKKokkos::init() ReactBirdKokkos::init(); - // do not allow recombination reactions (not supported) + // these two restrictions mirror the host (react_tce_qk.cpp:44-52) exactly -- + // same condition, same message. They are NOT Kokkos limitations: + // react tce/qk does not support recombination or compute_chem_rates on + // the CPU either, so lifting them here would diverge from the host. + // Only react tce supports both, and react_tce_kokkos.h ports that. for (int i = 0; i < nlist; i++) if (rlist[i].active && rlist[i].type == RECOMBINATION) diff --git a/src/KOKKOS/react_tce_qk_kokkos.h b/src/KOKKOS/react_tce_qk_kokkos.h index 01869f584..0cfe08b5f 100644 --- a/src/KOKKOS/react_tce_qk_kokkos.h +++ b/src/KOKKOS/react_tce_qk_kokkos.h @@ -119,11 +119,16 @@ int attempt_kk(Particle::OnePart *ip, Particle::OnePart *jp, eccq = pre_etrans + ip->evib; maxlev = static_cast (eccq * inverse_kT); if (eccq > r->d_coeff[1]) { + // sample into a local prob, not react_prob: react_prob feeds + // the "fired" test below, and a rejected draw must not leave + // a fractional value in it. Mirrors ReactTCEQK::attempt() + double prob = 0.0; do { iv = static_cast (rand_gen.drand()*(maxlev+0.99999999)); double evib = static_cast (iv / inverse_kT); - if (evib < eccq) react_prob = pow(1.0-evib/eccq,1.5-omega); - } while (rand_gen.drand() < react_prob); + if (evib < eccq) prob = pow(1.0-evib/eccq,1.5-omega); + else prob = 0.0; + } while (rand_gen.drand() < prob); ilevel = static_cast (fabs(r->d_coeff[4]) * inverse_kT); if (iv >= ilevel) react_prob = 1.0; } @@ -138,6 +143,7 @@ int attempt_kk(Particle::OnePart *ip, Particle::OnePart *jp, iv = rand_gen.drand()*(maxlev+0.99999999); double evib = static_cast (iv * boltz*d_species[mspec].vibtemp[0]); if (evib < eccq) prob = pow(1.0-evib/eccq,1.5 - r->d_coeff[6]); + else prob = 0.0; } while (rand_gen.drand() < prob); ilevel = static_cast (fabs(r->d_coeff[4]/boltz/d_species[mspec].vibtemp[0])); if (iv >= ilevel) react_prob = 1.0; diff --git a/src/KOKKOS/read_surf_kokkos.cpp b/src/KOKKOS/read_surf_kokkos.cpp index 4f273f12c..71f498ea6 100644 --- a/src/KOKKOS/read_surf_kokkos.cpp +++ b/src/KOKKOS/read_surf_kokkos.cpp @@ -39,7 +39,6 @@ enum{NEITHER,BAD,GOOD}; enum{NONE,CHECK,KEEP}; enum{UNKNOWN,OUTSIDE,INSIDE,OVERLAP}; // several files -#define MAXLINE 256 #define CHUNK 1024 #define EPSILON_NORM 1.0e-12 #define EPSILON_GRID 1.0e-3 diff --git a/src/KOKKOS/region_block_kokkos.h b/src/KOKKOS/region_block_kokkos.h index 5fa724f53..54317e666 100644 --- a/src/KOKKOS/region_block_kokkos.h +++ b/src/KOKKOS/region_block_kokkos.h @@ -42,6 +42,26 @@ class RegBlockKokkos : public RegBlock, public KokkosBase { void match_all_kokkos(DAT::tdual_int_1d) override; + // flatten to a single-token postfix stream; see region_prim_kokkos.h + + int flatten_region_kokkos(tdual_region_token_1d &k_tokens) override + { + region_token_grow(k_tokens,1); + RegionTokenKK &t = k_tokens.view_host()[0]; + t.type = RKK_TOK_PRIM; + RegionPrimKK &p = t.prim; + p.style = RKK_BLOCK; + p.interior = interior; + p.axis = 0; + p.a = p.b = p.c = p.d = p.e = p.f = 0.0; + p.n0 = p.n1 = p.n2 = 0.0; + p.a = xlo; p.b = xhi; p.c = ylo; p.d = yhi; p.e = zlo; p.f = zhi; + k_tokens.modify_host(); + k_tokens.sync_device(); + return 1; + } + + KOKKOS_INLINE_FUNCTION void operator()(TagRegBlockMatchAll, const int&) const; diff --git a/src/KOKKOS/region_cylinder_kokkos.h b/src/KOKKOS/region_cylinder_kokkos.h index f6f6a1e19..0cc484f03 100644 --- a/src/KOKKOS/region_cylinder_kokkos.h +++ b/src/KOKKOS/region_cylinder_kokkos.h @@ -42,6 +42,27 @@ class RegCylinderKokkos : public RegCylinder, public KokkosBase { void match_all_kokkos(DAT::tdual_int_1d) override; + // flatten to a single-token postfix stream; see region_prim_kokkos.h + + int flatten_region_kokkos(tdual_region_token_1d &k_tokens) override + { + region_token_grow(k_tokens,1); + RegionTokenKK &t = k_tokens.view_host()[0]; + t.type = RKK_TOK_PRIM; + RegionPrimKK &p = t.prim; + p.style = RKK_CYLINDER; + p.interior = interior; + p.axis = 0; + p.a = p.b = p.c = p.d = p.e = p.f = 0.0; + p.n0 = p.n1 = p.n2 = 0.0; + p.axis = (axis == 'x') ? 0 : ((axis == 'y') ? 1 : 2); + p.a = c1; p.b = c2; p.c = radius; p.d = lo; p.e = hi; + k_tokens.modify_host(); + k_tokens.sync_device(); + return 1; + } + + KOKKOS_INLINE_FUNCTION void operator()(TagRegCylinderMatchAll, const int&) const; diff --git a/src/KOKKOS/region_intersect_kokkos.cpp b/src/KOKKOS/region_intersect_kokkos.cpp new file mode 100644 index 000000000..e1b822e35 --- /dev/null +++ b/src/KOKKOS/region_intersect_kokkos.cpp @@ -0,0 +1,136 @@ +/* ---------------------------------------------------------------------- + SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer + http://sparta.github.io + Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov + Sandia National Laboratories + + Copyright (2014) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level SPARTA directory. +------------------------------------------------------------------------- */ + +#include "stdio.h" +#include "region_intersect_kokkos.h" +#include "domain.h" +#include "particle_kokkos.h" +#include "error.h" +#include "sparta_masks.h" + +using namespace SPARTA_NS; + +/* ---------------------------------------------------------------------- */ + +RegIntersectKokkos::RegIntersectKokkos(SPARTA *sparta, int narg, char **arg) : + RegIntersect(sparta, narg, arg) +{ + kokkos_flag = 1; + ntoken = 0; +} + +/* ---------------------------------------------------------------------- */ + +RegIntersectKokkos::~RegIntersectKokkos() +{ +} + +/* ---------------------------------------------------------------------- + flatten this region and its sub-regions into one device-resident postfix + (RPN) token stream: sub-stream(0) sub-stream(1) OP sub-stream(2) OP ... + followed by a NOT token when this composite is an exterior region + a sub-region contributes a whole sub-stream, so a sub-region may itself be + a region union or region intersect -- nesting is supported to any depth + whose evaluation fits the RKK_MAX_STACK boolean stack, which is checked + here on the host and errors out rather than truncating +------------------------------------------------------------------------- */ + +int RegIntersectKokkos::flatten_region_kokkos(tdual_region_token_1d &k_tokens_out) +{ + Region **regions = domain->regions; + + ntoken = 0; + + for (int i = 0; i < nregion; i++) { + Region *r = regions[list[i]]; + KokkosBase *rkk = dynamic_cast(r); + if (!rkk || !r->kokkos_flag) + error->all(FLERR,"KOKKOS package does not (yet) support the region style " + "used inside region intersect"); + + // k_sub is declared inside the loop on purpose: a composite sub-region + // hands back its own buffer, and a later primitive sub-region handed + // that same handle would write into it + + tdual_region_token_1d k_sub; + const int nsub = rkk->flatten_region_kokkos(k_sub); + if (nsub <= 0) + error->all(FLERR,"KOKKOS package does not (yet) support the region style " + "used inside region intersect"); + + // append the sub-region's stream, then the op that folds it into the + // running result (every sub-region past the first) + + region_token_grow(k_tokens,ntoken+nsub+2); + for (int j = 0; j < nsub; j++) + k_tokens.view_host()[ntoken++] = k_sub.view_host()[j]; + if (i) k_tokens.view_host()[ntoken++].type = RKK_TOK_INTERSECT; + } + + // this composite's own interior/exterior sense: !(hit ^ interior) + + if (!interior) { + region_token_grow(k_tokens,ntoken+1); + k_tokens.view_host()[ntoken++].type = RKK_TOK_NOT; + } + + // bound the boolean stack the kernel will need, on the host, before any + // particle is tested against the stream + + const int depth = region_token_depth(k_tokens,ntoken); + if (depth < 0) + error->all(FLERR,"Internal error flattening region intersect for the KOKKOS package"); + if (depth > RKK_MAX_STACK) { + char str[128]; + snprintf(str,sizeof(str),"Region intersect is nested too deeply for the KOKKOS package " + "(needs a boolean stack of %d, max is %d)",depth,RKK_MAX_STACK); + error->all(FLERR,str); + } + + k_tokens.modify_host(); + k_tokens.sync_device(); + + k_tokens_out = k_tokens; + return ntoken; +} + +/* ---------------------------------------------------------------------- */ + +void RegIntersectKokkos::match_all_kokkos(DAT::tdual_int_1d k_match_in) +{ + tdual_region_token_1d k_tokens_local; + const int ntoken_local = flatten_region_kokkos(k_tokens_local); + + d_match = k_match_in.view_device(); + ParticleKokkos* particleKK = (ParticleKokkos*) particle; + particleKK->sync(Device, PARTICLE_MASK); + d_particles = particleKK->k_particles.view_device(); + const int nlocal = particle->nlocal; + + auto l_tokens = k_tokens_local.view_device(); + auto l_match = d_match; + auto l_particles = d_particles; + const int l_ntoken = ntoken_local; + + copymode = 1; + Kokkos::parallel_for(Kokkos::RangePolicy(0,nlocal), + KOKKOS_LAMBDA(const int &i) { + const double x = l_particles[i].x[0]; + const double y = l_particles[i].x[1]; + const double z = l_particles[i].x[2]; + l_match[i] = region_match_kk(l_tokens,l_ntoken,x,y,z); + }); + copymode = 0; + k_match_in.modify_device(); +} diff --git a/src/KOKKOS/region_intersect_kokkos.h b/src/KOKKOS/region_intersect_kokkos.h new file mode 100644 index 000000000..efc062005 --- /dev/null +++ b/src/KOKKOS/region_intersect_kokkos.h @@ -0,0 +1,62 @@ +/* ---------------------------------------------------------------------- + SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer + http://sparta.github.io + Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov + Sandia National Laboratories + + Copyright (2014) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level SPARTA directory. +------------------------------------------------------------------------- */ + +#ifdef REGION_CLASS + +RegionStyle(intersect/kk,RegIntersectKokkos) + +#else + +#ifndef SPARTA_REGION_INTERSECT_KOKKOS_H +#define SPARTA_REGION_INTERSECT_KOKKOS_H + +#include "region_intersect.h" + +#include "kokkos_base.h" +#include "kokkos_type.h" + +namespace SPARTA_NS { + +// a composite region cannot dispatch to its sub-regions on the device, so it +// flattens them into a postfix (RPN) token stream instead; see +// region_prim_kokkos.h. each sub-region contributes its own whole +// sub-stream, so a sub-region may itself be a region union or region +// intersect: composites nest to arbitrary depth, bounded only by the +// RKK_MAX_STACK boolean stack depth checked here at flatten time. + +class RegIntersectKokkos : public RegIntersect, public KokkosBase { + + public: + typedef DeviceType device_type; + typedef ArrayTypes AT; + + RegIntersectKokkos(class SPARTA *, int, char **); + ~RegIntersectKokkos() override; + + void match_all_kokkos(DAT::tdual_int_1d) override; + int flatten_region_kokkos(tdual_region_token_1d &) override; + + private: + int groupbit; + typename AT::t_int_1d d_match; + t_particle_1d d_particles; + + tdual_region_token_1d k_tokens; // this region's own token stream + int ntoken; +}; + +} + +#endif +#endif diff --git a/src/KOKKOS/region_plane_kokkos.h b/src/KOKKOS/region_plane_kokkos.h index e42d7bb2d..5a4fd53d4 100644 --- a/src/KOKKOS/region_plane_kokkos.h +++ b/src/KOKKOS/region_plane_kokkos.h @@ -42,6 +42,27 @@ class RegPlaneKokkos : public RegPlane, public KokkosBase { void match_all_kokkos(DAT::tdual_int_1d) override; + // flatten to a single-token postfix stream; see region_prim_kokkos.h + + int flatten_region_kokkos(tdual_region_token_1d &k_tokens) override + { + region_token_grow(k_tokens,1); + RegionTokenKK &t = k_tokens.view_host()[0]; + t.type = RKK_TOK_PRIM; + RegionPrimKK &p = t.prim; + p.style = RKK_PLANE; + p.interior = interior; + p.axis = 0; + p.a = p.b = p.c = p.d = p.e = p.f = 0.0; + p.n0 = p.n1 = p.n2 = 0.0; + p.a = xp; p.b = yp; p.c = zp; + p.n0 = normal[0]; p.n1 = normal[1]; p.n2 = normal[2]; + k_tokens.modify_host(); + k_tokens.sync_device(); + return 1; + } + + KOKKOS_INLINE_FUNCTION void operator()(TagRegPlaneMatchAll, const int&) const; diff --git a/src/KOKKOS/region_prim_kokkos.h b/src/KOKKOS/region_prim_kokkos.h new file mode 100644 index 000000000..a94d0096f --- /dev/null +++ b/src/KOKKOS/region_prim_kokkos.h @@ -0,0 +1,231 @@ +/* ---------------------------------------------------------------------- + SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer + http://sparta.github.io + Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov + Sandia National Laboratories + + Copyright (2014) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level SPARTA directory. +------------------------------------------------------------------------- */ + +#ifndef SPARTA_REGION_PRIM_KOKKOS_H +#define SPARTA_REGION_PRIM_KOKKOS_H + +#include "kokkos_type.h" + +namespace SPARTA_NS { + +// Region::inside() is a host virtual, and virtual dispatch is not available +// inside a device kernel. Rather than have every consumer carry a KKCopy +// of each concrete region type and switch on a style string -- which is +// what update/emit used to do, and what capped the number of regions a +// run could use -- each Kokkos region flattens itself into a small +// device-resident POSTFIX (RPN) token stream, which a kernel walks with a +// fixed-depth boolean stack and no dispatch at all. +// +// a token is either +// RKK_TOK_PRIM push region_prim_match_kk(token.prim) onto the stack +// RKK_TOK_UNION pop 2, push (a || b) +// RKK_TOK_INTERSECT pop 2, push (a && b) +// RKK_TOK_NOT negate the top of stack +// +// a primitive region flattens to a single RKK_TOK_PRIM token whose own +// interior/exterior sense is already folded into the primitive (the +// !(inside ^ interior) of Region::match()). +// region union / region intersect flatten to the concatenation of their +// sub-regions' streams, left-folded with one op token after each +// sub-region past the first, followed by one RKK_TOK_NOT token when the +// composite itself is an exterior region (interior == 0), since +// !(hit ^ 0) == !hit and !(hit ^ 1) == hit. because a sub-region +// contributes a whole sub-stream rather than a single entry, composites +// nest to arbitrary depth as long as the stack bound below holds. +// +// an op token carries an unused RegionPrimKK payload. that wastes a little +// memory per op, but keeps the whole program in ONE view, so a consumer +// still holds a single DualView plus a token count. + +enum{RKK_BLOCK,RKK_CYLINDER,RKK_PLANE,RKK_SPHERE}; +enum{RKK_TOK_PRIM,RKK_TOK_UNION,RKK_TOK_INTERSECT,RKK_TOK_NOT}; + +// max boolean stack depth a token stream may require. a stream that needs +// more than this is rejected on the host at flatten time (see +// region_token_depth() below and its callers) -- never truncated. +// a flat composite of any number of primitives needs depth 2; depth D +// allows e.g. D-1 levels of right-nested composites. 16 is far past any +// region tree an input script is likely to build, and costs the kernel +// only a 16-int (64 byte) per-thread scratch array on the paths that are +// not the single-primitive fast path. + +enum{RKK_MAX_STACK = 16}; + +struct RegionPrimKK { + int style; // one of RKK_* + int interior; // this sub-region's own interior/exterior sense + int axis; // cylinder only: 0/1/2 for x/y/z + + // style-specific parameters, packed: + // BLOCK a..f = xlo,xhi,ylo,yhi,zlo,zhi + // CYLINDER a,b = c1,c2 c = radius d,e = lo,hi + // PLANE a,b,c = point n0,n1,n2 = normal + // SPHERE a,b,c = center d = radius + + double a,b,c,d,e,f; + double n0,n1,n2; +}; + +struct RegionTokenKK { + int type; // one of RKK_TOK_* + RegionPrimKK prim; // meaningful only when type == RKK_TOK_PRIM +}; + +typedef Kokkos::DualView + tdual_region_token_1d; +typedef tdual_region_token_1d::t_dev t_region_token_1d; + +/* ---------------------------------------------------------------------- + does x,y,z match a single flattened sub-region + mirrors Region::match(): !(inside ^ interior) +------------------------------------------------------------------------- */ + +KOKKOS_INLINE_FUNCTION +int region_prim_match_kk(const RegionPrimKK &p, + const double x, const double y, const double z) +{ + int inside = 0; + + if (p.style == RKK_BLOCK) { + if (x >= p.a && x <= p.b && y >= p.c && y <= p.d && z >= p.e && z <= p.f) + inside = 1; + + } else if (p.style == RKK_CYLINDER) { + double del1,del2; + if (p.axis == 0) { del1 = y - p.a; del2 = z - p.b; } + else if (p.axis == 1) { del1 = x - p.a; del2 = z - p.b; } + else { del1 = x - p.a; del2 = y - p.b; } + const double dist = sqrt(del1*del1 + del2*del2); + const double along = (p.axis == 0) ? x : ((p.axis == 1) ? y : z); + if (dist <= p.c && along >= p.d && along <= p.e) inside = 1; + + } else if (p.style == RKK_PLANE) { + const double dot = (x-p.a)*p.n0 + (y-p.b)*p.n1 + (z-p.c)*p.n2; + if (dot >= 0.0) inside = 1; + + } else { // RKK_SPHERE + const double delx = x - p.a; + const double dely = y - p.b; + const double delz = z - p.c; + if (sqrt(delx*delx + dely*dely + delz*delz) <= p.d) inside = 1; + } + + return !(inside ^ p.interior); +} + +/* ---------------------------------------------------------------------- + does x,y,z match a flattened region, i.e. evaluate its postfix token + stream against a small boolean stack + ntoken == 1 is the single-primitive fast path: no stack at all, exactly + the work the flat representation used to do + the stream is built and validated on the host (region_token_depth), so + the stack can never overflow or underflow here; the guards below are + belt-and-braces against a caller passing a stale ntoken +------------------------------------------------------------------------- */ + +template +KOKKOS_INLINE_FUNCTION +int region_match_kk(const ViewType &d_tokens, const int ntoken, + const double x, const double y, const double z) +{ + if (ntoken == 1) return region_prim_match_kk(d_tokens[0].prim,x,y,z); + if (ntoken <= 0) return 0; + + int stack[RKK_MAX_STACK]; + int nstack = 0; + + for (int i = 0; i < ntoken; i++) { + const int type = d_tokens[i].type; + + if (type == RKK_TOK_PRIM) { + if (nstack == RKK_MAX_STACK) return 0; + stack[nstack++] = region_prim_match_kk(d_tokens[i].prim,x,y,z); + + } else if (type == RKK_TOK_NOT) { + if (nstack < 1) return 0; + stack[nstack-1] = !stack[nstack-1]; + + } else { + if (nstack < 2) return 0; + const int b = stack[--nstack]; + const int a = stack[nstack-1]; + if (type == RKK_TOK_UNION) stack[nstack-1] = (a || b); + else stack[nstack-1] = (a && b); + } + } + + return stack[0]; +} + +/* ---------------------------------------------------------------------- + host-side helpers used while a composite region builds its token stream +------------------------------------------------------------------------- */ + +/* ---------------------------------------------------------------------- + make sure k_tokens can hold n tokens, preserving the tokens already in it + grows geometrically; the host view is the authoritative copy while a + stream is being built, so only host data is carried over +------------------------------------------------------------------------- */ + +inline void region_token_grow(tdual_region_token_1d &k_tokens, const int n) +{ + const int nmax = (int) k_tokens.extent(0); + if (nmax >= n) return; + + int nnew = nmax ? nmax : 8; + while (nnew < n) nnew *= 2; + + tdual_region_token_1d k_new("region:tokens",nnew); + for (int i = 0; i < nmax; i++) + k_new.view_host()[i] = k_tokens.view_host()[i]; + k_tokens = k_new; +} + +/* ---------------------------------------------------------------------- + boolean stack depth the first ntoken tokens of k_tokens will require + returns -1 if the stream is malformed (stack underflow, or anything + other than exactly one value left at the end) + host only: called at flatten time so a too-deep or ill-formed region tree + is an error, not a wrong answer in a kernel +------------------------------------------------------------------------- */ + +inline int region_token_depth(tdual_region_token_1d &k_tokens, const int ntoken) +{ + if (ntoken <= 0) return -1; + if ((int) k_tokens.extent(0) < ntoken) return -1; + + int nstack = 0; + int maxdepth = 0; + + for (int i = 0; i < ntoken; i++) { + const int type = k_tokens.view_host()[i].type; + + if (type == RKK_TOK_PRIM) { + nstack++; + if (nstack > maxdepth) maxdepth = nstack; + } else if (type == RKK_TOK_NOT) { + if (nstack < 1) return -1; + } else { + if (nstack < 2) return -1; + nstack--; + } + } + + if (nstack != 1) return -1; + return maxdepth; +} + +} + +#endif diff --git a/src/KOKKOS/region_sphere_kokkos.h b/src/KOKKOS/region_sphere_kokkos.h index a3f891ff4..b1e06776e 100644 --- a/src/KOKKOS/region_sphere_kokkos.h +++ b/src/KOKKOS/region_sphere_kokkos.h @@ -42,6 +42,26 @@ class RegSphereKokkos : public RegSphere, public KokkosBase { void match_all_kokkos(DAT::tdual_int_1d) override; + // flatten to a single-token postfix stream; see region_prim_kokkos.h + + int flatten_region_kokkos(tdual_region_token_1d &k_tokens) override + { + region_token_grow(k_tokens,1); + RegionTokenKK &t = k_tokens.view_host()[0]; + t.type = RKK_TOK_PRIM; + RegionPrimKK &p = t.prim; + p.style = RKK_SPHERE; + p.interior = interior; + p.axis = 0; + p.a = p.b = p.c = p.d = p.e = p.f = 0.0; + p.n0 = p.n1 = p.n2 = 0.0; + p.a = xc; p.b = yc; p.c = zc; p.d = radius; + k_tokens.modify_host(); + k_tokens.sync_device(); + return 1; + } + + KOKKOS_INLINE_FUNCTION void operator()(TagRegSphereMatchAll, const int&) const; diff --git a/src/KOKKOS/region_union_kokkos.cpp b/src/KOKKOS/region_union_kokkos.cpp new file mode 100644 index 000000000..334402f2c --- /dev/null +++ b/src/KOKKOS/region_union_kokkos.cpp @@ -0,0 +1,136 @@ +/* ---------------------------------------------------------------------- + SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer + http://sparta.github.io + Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov + Sandia National Laboratories + + Copyright (2014) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level SPARTA directory. +------------------------------------------------------------------------- */ + +#include "stdio.h" +#include "region_union_kokkos.h" +#include "domain.h" +#include "particle_kokkos.h" +#include "error.h" +#include "sparta_masks.h" + +using namespace SPARTA_NS; + +/* ---------------------------------------------------------------------- */ + +RegUnionKokkos::RegUnionKokkos(SPARTA *sparta, int narg, char **arg) : + RegUnion(sparta, narg, arg) +{ + kokkos_flag = 1; + ntoken = 0; +} + +/* ---------------------------------------------------------------------- */ + +RegUnionKokkos::~RegUnionKokkos() +{ +} + +/* ---------------------------------------------------------------------- + flatten this region and its sub-regions into one device-resident postfix + (RPN) token stream: sub-stream(0) sub-stream(1) OP sub-stream(2) OP ... + followed by a NOT token when this composite is an exterior region + a sub-region contributes a whole sub-stream, so a sub-region may itself be + a region union or region intersect -- nesting is supported to any depth + whose evaluation fits the RKK_MAX_STACK boolean stack, which is checked + here on the host and errors out rather than truncating +------------------------------------------------------------------------- */ + +int RegUnionKokkos::flatten_region_kokkos(tdual_region_token_1d &k_tokens_out) +{ + Region **regions = domain->regions; + + ntoken = 0; + + for (int i = 0; i < nregion; i++) { + Region *r = regions[list[i]]; + KokkosBase *rkk = dynamic_cast(r); + if (!rkk || !r->kokkos_flag) + error->all(FLERR,"KOKKOS package does not (yet) support the region style " + "used inside region union"); + + // k_sub is declared inside the loop on purpose: a composite sub-region + // hands back its own buffer, and a later primitive sub-region handed + // that same handle would write into it + + tdual_region_token_1d k_sub; + const int nsub = rkk->flatten_region_kokkos(k_sub); + if (nsub <= 0) + error->all(FLERR,"KOKKOS package does not (yet) support the region style " + "used inside region union"); + + // append the sub-region's stream, then the op that folds it into the + // running result (every sub-region past the first) + + region_token_grow(k_tokens,ntoken+nsub+2); + for (int j = 0; j < nsub; j++) + k_tokens.view_host()[ntoken++] = k_sub.view_host()[j]; + if (i) k_tokens.view_host()[ntoken++].type = RKK_TOK_UNION; + } + + // this composite's own interior/exterior sense: !(hit ^ interior) + + if (!interior) { + region_token_grow(k_tokens,ntoken+1); + k_tokens.view_host()[ntoken++].type = RKK_TOK_NOT; + } + + // bound the boolean stack the kernel will need, on the host, before any + // particle is tested against the stream + + const int depth = region_token_depth(k_tokens,ntoken); + if (depth < 0) + error->all(FLERR,"Internal error flattening region union for the KOKKOS package"); + if (depth > RKK_MAX_STACK) { + char str[128]; + snprintf(str,sizeof(str),"Region union is nested too deeply for the KOKKOS package " + "(needs a boolean stack of %d, max is %d)",depth,RKK_MAX_STACK); + error->all(FLERR,str); + } + + k_tokens.modify_host(); + k_tokens.sync_device(); + + k_tokens_out = k_tokens; + return ntoken; +} + +/* ---------------------------------------------------------------------- */ + +void RegUnionKokkos::match_all_kokkos(DAT::tdual_int_1d k_match_in) +{ + tdual_region_token_1d k_tokens_local; + const int ntoken_local = flatten_region_kokkos(k_tokens_local); + + d_match = k_match_in.view_device(); + ParticleKokkos* particleKK = (ParticleKokkos*) particle; + particleKK->sync(Device, PARTICLE_MASK); + d_particles = particleKK->k_particles.view_device(); + const int nlocal = particle->nlocal; + + auto l_tokens = k_tokens_local.view_device(); + auto l_match = d_match; + auto l_particles = d_particles; + const int l_ntoken = ntoken_local; + + copymode = 1; + Kokkos::parallel_for(Kokkos::RangePolicy(0,nlocal), + KOKKOS_LAMBDA(const int &i) { + const double x = l_particles[i].x[0]; + const double y = l_particles[i].x[1]; + const double z = l_particles[i].x[2]; + l_match[i] = region_match_kk(l_tokens,l_ntoken,x,y,z); + }); + copymode = 0; + k_match_in.modify_device(); +} diff --git a/src/KOKKOS/region_union_kokkos.h b/src/KOKKOS/region_union_kokkos.h new file mode 100644 index 000000000..2ddbf4660 --- /dev/null +++ b/src/KOKKOS/region_union_kokkos.h @@ -0,0 +1,62 @@ +/* ---------------------------------------------------------------------- + SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer + http://sparta.github.io + Steve Plimpton, sjplimp@gmail.com, Michael Gallis, magalli@sandia.gov + Sandia National Laboratories + + Copyright (2014) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level SPARTA directory. +------------------------------------------------------------------------- */ + +#ifdef REGION_CLASS + +RegionStyle(union/kk,RegUnionKokkos) + +#else + +#ifndef SPARTA_REGION_UNION_KOKKOS_H +#define SPARTA_REGION_UNION_KOKKOS_H + +#include "region_union.h" + +#include "kokkos_base.h" +#include "kokkos_type.h" + +namespace SPARTA_NS { + +// a composite region cannot dispatch to its sub-regions on the device, so it +// flattens them into a postfix (RPN) token stream instead; see +// region_prim_kokkos.h. each sub-region contributes its own whole +// sub-stream, so a sub-region may itself be a region union or region +// intersect: composites nest to arbitrary depth, bounded only by the +// RKK_MAX_STACK boolean stack depth checked here at flatten time. + +class RegUnionKokkos : public RegUnion, public KokkosBase { + + public: + typedef DeviceType device_type; + typedef ArrayTypes AT; + + RegUnionKokkos(class SPARTA *, int, char **); + ~RegUnionKokkos() override; + + void match_all_kokkos(DAT::tdual_int_1d) override; + int flatten_region_kokkos(tdual_region_token_1d &) override; + + private: + int groupbit; + typename AT::t_int_1d d_match; + t_particle_1d d_particles; + + tdual_region_token_1d k_tokens; // this region's own token stream + int ntoken; +}; + +} + +#endif +#endif diff --git a/src/KOKKOS/surf_collide_adiabatic_kokkos.cpp b/src/KOKKOS/surf_collide_adiabatic_kokkos.cpp index 16cb6381a..668151754 100644 --- a/src/KOKKOS/surf_collide_adiabatic_kokkos.cpp +++ b/src/KOKKOS/surf_collide_adiabatic_kokkos.cpp @@ -45,8 +45,11 @@ SurfCollideAdiabaticKokkos::SurfCollideAdiabaticKokkos(SPARTA *sparta, int narg, SurfCollideAdiabatic(sparta, narg, arg), fix_ambi_kk_copy(sparta), fix_vibmode_kk_copy(sparta), +#ifdef SPARTA_KOKKOS_FIXED_LISTS sr_kk_global_copy{VAL_2(KKCopy(sparta))}, sr_kk_prob_copy{VAL_2(KKCopy(sparta))}, + sr_kk_adsorb_copy{VAL_2(KKCopy(sparta))}, +#endif rand_pool(12345 + comm->me #ifdef SPARTA_KOKKOS_EXACT , sparta @@ -82,8 +85,11 @@ SurfCollideAdiabaticKokkos::SurfCollideAdiabaticKokkos(SPARTA *sparta) : SurfCollideAdiabatic(sparta), fix_ambi_kk_copy(sparta), fix_vibmode_kk_copy(sparta), +#ifdef SPARTA_KOKKOS_FIXED_LISTS sr_kk_global_copy{VAL_2(KKCopy(sparta))}, sr_kk_prob_copy{VAL_2(KKCopy(sparta))}, + sr_kk_adsorb_copy{VAL_2(KKCopy(sparta))}, +#endif rand_pool(12345 // seed doesn't matter since it will just be copied over #ifdef SPARTA_KOKKOS_EXACT , sparta @@ -151,36 +157,82 @@ void SurfCollideAdiabaticKokkos::pre_collide() fix_vibmode_kk_copy.copy(vfix_kk); } +#ifdef SPARTA_KOKKOS_FIXED_LISTS if (surf->nsr > KOKKOS_MAX_TOT_SURF_REACT) - error->all(FLERR,"Kokkos currently supports two instances of each surface reaction method"); + error->all(FLERR,"Kokkos currently supports a limited number of surface reaction methods"); +#else + + // the buffers must be sized before anything is blitted into them. surf->nsr + // bounds the count of every individual style, so sizing all of them to it + // needs no counting pass, and the loop below still runs pre_react() in + // surf react list order + + sr_idx_resize(k_sr_type_list,d_sr_type_list,surf->nsr); + sr_idx_resize(k_sr_map,d_sr_map,surf->nsr); + sr_buf_resize(k_sr_global,d_sr_global,surf->nsr); + sr_buf_resize(k_sr_prob,d_sr_prob,surf->nsr); + sr_buf_resize(k_sr_adsorb,d_sr_adsorb,surf->nsr); +#endif if (surf->nsr > 0) { - int nglob,nprob; - nglob = nprob = 0; + int nglob,nprob,nadsorb; + nglob = nprob = nadsorb = 0; for (int n = 0; n < surf->nsr; n++) { if (!surf->sr[n]->kokkosable) error->all(FLERR,"Must use Kokkos-enabled surface reaction method with Kokkos"); if (strcmp(surf->sr[n]->style,"global") == 0) { +#ifdef SPARTA_KOKKOS_FIXED_LISTS if (nglob >= KOKKOS_MAX_SURF_REACT_PER_TYPE) error->all(FLERR,"Kokkos currently supports two instances of each surface reaction method"); sr_kk_global_copy[nglob].copy((SurfReactGlobalKokkos*)(surf->sr[n])); - sr_kk_global_copy[nglob].obj.pre_react(); - sr_type_list[n] = 0; - sr_map[n] = nglob; +#else + sr_buf_blit(k_sr_global,nglob,(SurfReactGlobalKokkos*)(surf->sr[n])); +#endif + KK_SR_H_GLOBAL(nglob).pre_react(); + KK_SR_H_TYPE(n) = 0; + KK_SR_H_MAP(n) = nglob; nglob++; } else if (strcmp(surf->sr[n]->style,"prob") == 0) { +#ifdef SPARTA_KOKKOS_FIXED_LISTS if (nprob >= KOKKOS_MAX_SURF_REACT_PER_TYPE) error->all(FLERR,"Kokkos currently supports two instances of each surface reaction method"); sr_kk_prob_copy[nprob].copy((SurfReactProbKokkos*)(surf->sr[n])); - sr_kk_prob_copy[nprob].obj.pre_react(); - sr_type_list[n] = 1; - sr_map[n] = nprob; +#else + sr_buf_blit(k_sr_prob,nprob,(SurfReactProbKokkos*)(surf->sr[n])); +#endif + KK_SR_H_PROB(nprob).pre_react(); + KK_SR_H_TYPE(n) = 1; + KK_SR_H_MAP(n) = nprob; nprob++; + } else if (strcmp(surf->sr[n]->style,"adsorb") == 0) { +#ifdef SPARTA_KOKKOS_FIXED_LISTS + if (nadsorb >= KOKKOS_MAX_SURF_REACT_PER_TYPE) + error->all(FLERR,"Kokkos currently supports two instances of each surface reaction method"); + sr_kk_adsorb_copy[nadsorb].copy((SurfReactAdsorbKokkos*)(surf->sr[n])); +#else + sr_buf_blit(k_sr_adsorb,nadsorb,(SurfReactAdsorbKokkos*)(surf->sr[n])); +#endif + KK_SR_H_ADSORB(nadsorb).pre_react(); + KK_SR_H_TYPE(n) = 2; + KK_SR_H_MAP(n) = nadsorb; + nadsorb++; } else { - error->all(FLERR,"This Kokkos surf_collide style supports only surf_react global/prob; surf_react adsorb requires surf_collide cll"); + error->all(FLERR,"Unknown Kokkos surface reaction method"); } } +#ifndef SPARTA_KOKKOS_FIXED_LISTS + + // the models were blitted into the host image of the buffers and their + // pre_react() ran there; push the result to the device + + sr_buf_sync(k_sr_global,d_sr_global); + sr_buf_sync(k_sr_prob,d_sr_prob); + sr_buf_sync(k_sr_adsorb,d_sr_adsorb); + sr_idx_sync(k_sr_type_list,d_sr_type_list); + sr_idx_sync(k_sr_map,d_sr_map); +#endif + } if (random == NULL) { @@ -226,8 +278,9 @@ void SurfCollideAdiabaticKokkos::post_collide() // would be orphaned and its allocation never freed for (int n = 0; n < surf->nsr; n++) { - if (sr_type_list[n] == 0) sr_kk_global_copy[sr_map[n]].obj.post_react(); - else sr_kk_prob_copy[sr_map[n]].obj.post_react(); + if (KK_SR_H_TYPE(n) == 0) KK_SR_H_GLOBAL(KK_SR_H_MAP(n)).post_react(); + else if (KK_SR_H_TYPE(n) == 1) KK_SR_H_PROB(KK_SR_H_MAP(n)).post_react(); + else KK_SR_H_ADSORB(KK_SR_H_MAP(n)).post_react(); } } @@ -239,17 +292,31 @@ void SurfCollideAdiabaticKokkos::backup() d_particles = particle_kk->k_particles.view_device(); if (surf->nsr > 0) { - int nglob,nprob; - nglob = nprob = 0; + int nglob,nprob,nadsorb; + nglob = nprob = nadsorb = 0; for (int n = 0; n < surf->nsr; n++) { if (strcmp(surf->sr[n]->style,"global") == 0) { - sr_kk_global_copy[nglob].obj.backup(); + KK_SR_H_GLOBAL(nglob).backup(); nglob++; } else if (strcmp(surf->sr[n]->style,"prob") == 0) { - sr_kk_prob_copy[nprob].obj.backup(); + KK_SR_H_PROB(nprob).backup(); nprob++; + } else if (strcmp(surf->sr[n]->style,"adsorb") == 0) { + KK_SR_H_ADSORB(nadsorb).backup(); + nadsorb++; } } + +#ifndef SPARTA_KOKKOS_FIXED_LISTS + + // backup() rewrites members of each model -- d_particles above all, which + // a grow reallocates -- so the device image of the buffers is stale + // until it is pushed again + + sr_buf_sync(k_sr_global,d_sr_global); + sr_buf_sync(k_sr_prob,d_sr_prob); + sr_buf_sync(k_sr_adsorb,d_sr_adsorb); +#endif } #ifdef SPARTA_KOKKOS_EXACT @@ -264,17 +331,31 @@ void SurfCollideAdiabaticKokkos::backup() void SurfCollideAdiabaticKokkos::restore() { if (surf->nsr > 0) { - int nglob,nprob; - nglob = nprob = 0; + int nglob,nprob,nadsorb; + nglob = nprob = nadsorb = 0; for (int n = 0; n < surf->nsr; n++) { if (strcmp(surf->sr[n]->style,"global") == 0) { - sr_kk_global_copy[nglob].obj.restore(); + KK_SR_H_GLOBAL(nglob).restore(); nglob++; } else if (strcmp(surf->sr[n]->style,"prob") == 0) { - sr_kk_prob_copy[nprob].obj.restore(); + KK_SR_H_PROB(nprob).restore(); nprob++; + } else if (strcmp(surf->sr[n]->style,"adsorb") == 0) { + KK_SR_H_ADSORB(nadsorb).restore(); + nadsorb++; } } + +#ifndef SPARTA_KOKKOS_FIXED_LISTS + + // restore() writes no member of a model today, only deep_copies into Views + // the model already holds, but the buffers are pushed for the same reason + // backup() pushes them: the device image must never trail the host one + + sr_buf_sync(k_sr_global,d_sr_global); + sr_buf_sync(k_sr_prob,d_sr_prob); + sr_buf_sync(k_sr_adsorb,d_sr_adsorb); +#endif } Kokkos::deep_copy(d_scalars,0); diff --git a/src/KOKKOS/surf_collide_adiabatic_kokkos.h b/src/KOKKOS/surf_collide_adiabatic_kokkos.h index e34f05dad..5d7ef0245 100644 --- a/src/KOKKOS/surf_collide_adiabatic_kokkos.h +++ b/src/KOKKOS/surf_collide_adiabatic_kokkos.h @@ -31,6 +31,7 @@ SurfCollideStyle(adiabatic/kk,SurfCollideAdiabaticKokkos) #include "fix_vibmode_kokkos.h" #include "surf_react_global_kokkos.h" #include "surf_react_prob_kokkos.h" +#include "surf_react_adsorb_kokkos.h" namespace SPARTA_NS { @@ -84,10 +85,23 @@ class SurfCollideAdiabaticKokkos : public SurfCollideAdiabatic { KKCopy fix_ambi_kk_copy; KKCopy fix_vibmode_kk_copy; + // the active surf react models this model may dispatch to, partitioned by + // style. Two representations, selected by SPARTA_KOKKOS_FIXED_LISTS (see + // kokkos_type.h); the device dispatch site in collide_kokkos() below is + // written once, against the KK_SR_* accessors defined there. + +#ifdef SPARTA_KOKKOS_FIXED_LISTS int sr_type_list[KOKKOS_MAX_TOT_SURF_REACT]; int sr_map[KOKKOS_MAX_TOT_SURF_REACT]; KKCopy sr_kk_global_copy[KOKKOS_MAX_SURF_REACT_PER_TYPE]; KKCopy sr_kk_prob_copy[KOKKOS_MAX_SURF_REACT_PER_TYPE]; + KKCopy sr_kk_adsorb_copy[KOKKOS_MAX_SURF_REACT_PER_TYPE]; +#else + DAT::tdual_int_1d k_sr_type_list,k_sr_map; + DAT::t_int_1d d_sr_type_list,d_sr_map; + DAT::tdual_char_1d k_sr_global,k_sr_prob,k_sr_adsorb; + DAT::t_char_1d d_sr_global,d_sr_prob,d_sr_adsorb; +#endif public: @@ -131,14 +145,17 @@ class SurfCollideAdiabaticKokkos : public SurfCollideAdiabatic { if (REACT && isr >= 0) { if (ambi_flag || vibmode_flag) memcpy(&iorig,ip,sizeof(Particle::OnePart)); - int sr_type = sr_type_list[isr]; - int m = sr_map[isr]; + int sr_type = KK_SR_TYPE(isr); + int m = KK_SR_MAP(isr); if (sr_type == 0) { - reaction = sr_kk_global_copy[m].obj. + reaction = KK_SR_GLOBAL(m). react_kokkos(ip,isurf,norm,jp,velreset,d_retry,d_nlocal); } else if (sr_type == 1) { - reaction = sr_kk_prob_copy[m].obj. + reaction = KK_SR_PROB(m). + react_kokkos(ip,isurf,norm,jp,velreset,d_retry,d_nlocal); + } else if (sr_type == 2) { + reaction = KK_SR_ADSORB(m). react_kokkos(ip,isurf,norm,jp,velreset,d_retry,d_nlocal); } diff --git a/src/KOKKOS/surf_collide_cll_kokkos.cpp b/src/KOKKOS/surf_collide_cll_kokkos.cpp index 4766b78f6..51db19334 100644 --- a/src/KOKKOS/surf_collide_cll_kokkos.cpp +++ b/src/KOKKOS/surf_collide_cll_kokkos.cpp @@ -49,9 +49,11 @@ SurfCollideCLLKokkos::SurfCollideCLLKokkos(SPARTA *sparta, int narg, char **arg) SurfCollideCLL(sparta, narg, arg), fix_ambi_kk_copy(sparta), fix_vibmode_kk_copy(sparta), +#ifdef SPARTA_KOKKOS_FIXED_LISTS sr_kk_global_copy{VAL_2(KKCopy(sparta))}, sr_kk_prob_copy{VAL_2(KKCopy(sparta))}, sr_kk_adsorb_copy{VAL_2(KKCopy(sparta))}, +#endif rand_pool(12345 + comm->me #ifdef SPARTA_KOKKOS_EXACT , sparta @@ -87,9 +89,11 @@ SurfCollideCLLKokkos::SurfCollideCLLKokkos(SPARTA *sparta) : SurfCollideCLL(sparta), fix_ambi_kk_copy(sparta), fix_vibmode_kk_copy(sparta), +#ifdef SPARTA_KOKKOS_FIXED_LISTS sr_kk_global_copy{VAL_2(KKCopy(sparta))}, sr_kk_prob_copy{VAL_2(KKCopy(sparta))}, sr_kk_adsorb_copy{VAL_2(KKCopy(sparta))}, +#endif rand_pool(12345 // seed doesn't matter since it will just be copied over #ifdef SPARTA_KOKKOS_EXACT , sparta @@ -238,8 +242,22 @@ void SurfCollideCLLKokkos::pre_collide() fix_vibmode_kk_copy.copy(vfix_kk); } +#ifdef SPARTA_KOKKOS_FIXED_LISTS if (surf->nsr > KOKKOS_MAX_TOT_SURF_REACT) error->all(FLERR,"Kokkos currently supports two instances of each surface reaction method"); +#else + + // the buffers must be sized before anything is blitted into them. surf->nsr + // bounds the count of every individual style, so sizing all of them to it + // needs no counting pass, and the loop below still runs pre_react() in + // surf react list order + + sr_idx_resize(k_sr_type_list,d_sr_type_list,surf->nsr); + sr_idx_resize(k_sr_map,d_sr_map,surf->nsr); + sr_buf_resize(k_sr_global,d_sr_global,surf->nsr); + sr_buf_resize(k_sr_prob,d_sr_prob,surf->nsr); + sr_buf_resize(k_sr_adsorb,d_sr_adsorb,surf->nsr); +#endif if (surf->nsr > 0) { int nglob,nprob,nadsorb; @@ -248,34 +266,58 @@ void SurfCollideCLLKokkos::pre_collide() if (!surf->sr[n]->kokkosable) error->all(FLERR,"Must use Kokkos-enabled surface reaction method with Kokkos"); if (strcmp(surf->sr[n]->style,"global") == 0) { +#ifdef SPARTA_KOKKOS_FIXED_LISTS if (nglob >= KOKKOS_MAX_SURF_REACT_PER_TYPE) error->all(FLERR,"Kokkos currently supports two instances of each surface reaction method"); sr_kk_global_copy[nglob].copy((SurfReactGlobalKokkos*)(surf->sr[n])); - sr_kk_global_copy[nglob].obj.pre_react(); - sr_type_list[n] = 0; - sr_map[n] = nglob; +#else + sr_buf_blit(k_sr_global,nglob,(SurfReactGlobalKokkos*)(surf->sr[n])); +#endif + KK_SR_H_GLOBAL(nglob).pre_react(); + KK_SR_H_TYPE(n) = 0; + KK_SR_H_MAP(n) = nglob; nglob++; } else if (strcmp(surf->sr[n]->style,"prob") == 0) { +#ifdef SPARTA_KOKKOS_FIXED_LISTS if (nprob >= KOKKOS_MAX_SURF_REACT_PER_TYPE) error->all(FLERR,"Kokkos currently supports two instances of each surface reaction method"); sr_kk_prob_copy[nprob].copy((SurfReactProbKokkos*)(surf->sr[n])); - sr_kk_prob_copy[nprob].obj.pre_react(); - sr_type_list[n] = 1; - sr_map[n] = nprob; +#else + sr_buf_blit(k_sr_prob,nprob,(SurfReactProbKokkos*)(surf->sr[n])); +#endif + KK_SR_H_PROB(nprob).pre_react(); + KK_SR_H_TYPE(n) = 1; + KK_SR_H_MAP(n) = nprob; nprob++; } else if (strcmp(surf->sr[n]->style,"adsorb") == 0) { +#ifdef SPARTA_KOKKOS_FIXED_LISTS if (nadsorb >= KOKKOS_MAX_SURF_REACT_PER_TYPE) error->all(FLERR,"Kokkos currently supports two instances of each surface reaction method"); sr_kk_adsorb_copy[nadsorb].copy((SurfReactAdsorbKokkos*)(surf->sr[n])); - sr_kk_adsorb_copy[nadsorb].obj.pre_react(); - sr_type_list[n] = 2; - sr_map[n] = nadsorb; +#else + sr_buf_blit(k_sr_adsorb,nadsorb,(SurfReactAdsorbKokkos*)(surf->sr[n])); +#endif + KK_SR_H_ADSORB(nadsorb).pre_react(); + KK_SR_H_TYPE(n) = 2; + KK_SR_H_MAP(n) = nadsorb; nadsorb++; } else { error->all(FLERR,"Unknown Kokkos surface reaction method"); } } +#ifndef SPARTA_KOKKOS_FIXED_LISTS + + // the models were blitted into the host image of the buffers and their + // pre_react() ran there; push the result to the device + + sr_buf_sync(k_sr_global,d_sr_global); + sr_buf_sync(k_sr_prob,d_sr_prob); + sr_buf_sync(k_sr_adsorb,d_sr_adsorb); + sr_idx_sync(k_sr_type_list,d_sr_type_list); + sr_idx_sync(k_sr_map,d_sr_map); +#endif + } if (random == NULL) { @@ -327,9 +369,9 @@ void SurfCollideCLLKokkos::post_collide() // would be orphaned and its allocation never freed for (int n = 0; n < surf->nsr; n++) { - if (sr_type_list[n] == 0) sr_kk_global_copy[sr_map[n]].obj.post_react(); - else if (sr_type_list[n] == 1) sr_kk_prob_copy[sr_map[n]].obj.post_react(); - else sr_kk_adsorb_copy[sr_map[n]].obj.post_react(); + if (KK_SR_H_TYPE(n) == 0) KK_SR_H_GLOBAL(KK_SR_H_MAP(n)).post_react(); + else if (KK_SR_H_TYPE(n) == 1) KK_SR_H_PROB(KK_SR_H_MAP(n)).post_react(); + else KK_SR_H_ADSORB(KK_SR_H_MAP(n)).post_react(); } } @@ -345,16 +387,27 @@ void SurfCollideCLLKokkos::backup() nglob = nprob = nadsorb = 0; for (int n = 0; n < surf->nsr; n++) { if (strcmp(surf->sr[n]->style,"global") == 0) { - sr_kk_global_copy[nglob].obj.backup(); + KK_SR_H_GLOBAL(nglob).backup(); nglob++; } else if (strcmp(surf->sr[n]->style,"prob") == 0) { - sr_kk_prob_copy[nprob].obj.backup(); + KK_SR_H_PROB(nprob).backup(); nprob++; } else if (strcmp(surf->sr[n]->style,"adsorb") == 0) { - sr_kk_adsorb_copy[nadsorb].obj.backup(); + KK_SR_H_ADSORB(nadsorb).backup(); nadsorb++; } } + +#ifndef SPARTA_KOKKOS_FIXED_LISTS + + // backup() rewrites members of each model -- d_particles above all, which + // a grow reallocates -- so the device image of the buffers is stale + // until it is pushed again + + sr_buf_sync(k_sr_global,d_sr_global); + sr_buf_sync(k_sr_prob,d_sr_prob); + sr_buf_sync(k_sr_adsorb,d_sr_adsorb); +#endif } #ifdef SPARTA_KOKKOS_EXACT @@ -373,16 +426,27 @@ void SurfCollideCLLKokkos::restore() nglob = nprob = nadsorb = 0; for (int n = 0; n < surf->nsr; n++) { if (strcmp(surf->sr[n]->style,"global") == 0) { - sr_kk_global_copy[nglob].obj.restore(); + KK_SR_H_GLOBAL(nglob).restore(); nglob++; } else if (strcmp(surf->sr[n]->style,"prob") == 0) { - sr_kk_prob_copy[nprob].obj.restore(); + KK_SR_H_PROB(nprob).restore(); nprob++; } else if (strcmp(surf->sr[n]->style,"adsorb") == 0) { - sr_kk_adsorb_copy[nadsorb].obj.restore(); + KK_SR_H_ADSORB(nadsorb).restore(); nadsorb++; } } + +#ifndef SPARTA_KOKKOS_FIXED_LISTS + + // restore() writes no member of a model today, only deep_copies into Views + // the model already holds, but the buffers are pushed for the same reason + // backup() pushes them: the device image must never trail the host one + + sr_buf_sync(k_sr_global,d_sr_global); + sr_buf_sync(k_sr_prob,d_sr_prob); + sr_buf_sync(k_sr_adsorb,d_sr_adsorb); +#endif } Kokkos::deep_copy(d_scalars,0); diff --git a/src/KOKKOS/surf_collide_cll_kokkos.h b/src/KOKKOS/surf_collide_cll_kokkos.h index ba6e6832a..4dbe67792 100644 --- a/src/KOKKOS/surf_collide_cll_kokkos.h +++ b/src/KOKKOS/surf_collide_cll_kokkos.h @@ -91,11 +91,23 @@ class SurfCollideCLLKokkos : public SurfCollideCLL { KKCopy fix_ambi_kk_copy; KKCopy fix_vibmode_kk_copy; + // the active surf react models this model may dispatch to, partitioned by + // style. Two representations, selected by SPARTA_KOKKOS_FIXED_LISTS (see + // kokkos_type.h); the device dispatch site in collide_kokkos() below is + // written once, against the KK_SR_* accessors defined there. + +#ifdef SPARTA_KOKKOS_FIXED_LISTS int sr_type_list[KOKKOS_MAX_TOT_SURF_REACT]; int sr_map[KOKKOS_MAX_TOT_SURF_REACT]; KKCopy sr_kk_global_copy[KOKKOS_MAX_SURF_REACT_PER_TYPE]; KKCopy sr_kk_prob_copy[KOKKOS_MAX_SURF_REACT_PER_TYPE]; KKCopy sr_kk_adsorb_copy[KOKKOS_MAX_SURF_REACT_PER_TYPE]; +#else + DAT::tdual_int_1d k_sr_type_list,k_sr_map; + DAT::t_int_1d d_sr_type_list,d_sr_map; + DAT::tdual_char_1d k_sr_global,k_sr_prob,k_sr_adsorb; + DAT::t_char_1d d_sr_global,d_sr_prob,d_sr_adsorb; +#endif public: @@ -134,17 +146,17 @@ class SurfCollideCLLKokkos : public SurfCollideCLL { if (REACT && isr >= 0) { if (ambi_flag || vibmode_flag) memcpy(&iorig,ip,sizeof(Particle::OnePart)); - int sr_type = sr_type_list[isr]; - int m = sr_map[isr]; + int sr_type = KK_SR_TYPE(isr); + int m = KK_SR_MAP(isr); if (sr_type == 0) { - reaction = sr_kk_global_copy[m].obj. + reaction = KK_SR_GLOBAL(m). react_kokkos(ip,isurf,norm,jp,velreset,d_retry,d_nlocal); } else if (sr_type == 1) { - reaction = sr_kk_prob_copy[m].obj. + reaction = KK_SR_PROB(m). react_kokkos(ip,isurf,norm,jp,velreset,d_retry,d_nlocal); } else if (sr_type == 2) { - reaction = sr_kk_adsorb_copy[m].obj. + reaction = KK_SR_ADSORB(m). react_kokkos(ip,isurf,norm,jp,velreset,d_retry,d_nlocal); } diff --git a/src/KOKKOS/surf_collide_diffuse_kokkos.cpp b/src/KOKKOS/surf_collide_diffuse_kokkos.cpp index b46a65926..fc85d38be 100644 --- a/src/KOKKOS/surf_collide_diffuse_kokkos.cpp +++ b/src/KOKKOS/surf_collide_diffuse_kokkos.cpp @@ -48,8 +48,11 @@ SurfCollideDiffuseKokkos::SurfCollideDiffuseKokkos(SPARTA *sparta, int narg, cha SurfCollideDiffuse(sparta, narg, arg), fix_ambi_kk_copy(sparta), fix_vibmode_kk_copy(sparta), +#ifdef SPARTA_KOKKOS_FIXED_LISTS sr_kk_global_copy{VAL_2(KKCopy(sparta))}, sr_kk_prob_copy{VAL_2(KKCopy(sparta))}, + sr_kk_adsorb_copy{VAL_2(KKCopy(sparta))}, +#endif rand_pool(12345 + comm->me #ifdef SPARTA_KOKKOS_EXACT , sparta @@ -85,8 +88,11 @@ SurfCollideDiffuseKokkos::SurfCollideDiffuseKokkos(SPARTA *sparta) : SurfCollideDiffuse(sparta), fix_ambi_kk_copy(sparta), fix_vibmode_kk_copy(sparta), +#ifdef SPARTA_KOKKOS_FIXED_LISTS sr_kk_global_copy{VAL_2(KKCopy(sparta))}, sr_kk_prob_copy{VAL_2(KKCopy(sparta))}, + sr_kk_adsorb_copy{VAL_2(KKCopy(sparta))}, +#endif rand_pool(12345 // seed doesn't matter since it will just be copied over #ifdef SPARTA_KOKKOS_EXACT , sparta @@ -235,35 +241,81 @@ void SurfCollideDiffuseKokkos::pre_collide() fix_vibmode_kk_copy.copy(vfix_kk); } +#ifdef SPARTA_KOKKOS_FIXED_LISTS if (surf->nsr > KOKKOS_MAX_TOT_SURF_REACT) error->all(FLERR,"Kokkos currently supports a limited number of surface reaction methods"); +#else + + // the buffers must be sized before anything is blitted into them. surf->nsr + // bounds the count of every individual style, so sizing all of them to it + // needs no counting pass, and the loop below still runs pre_react() in + // surf react list order + + sr_idx_resize(k_sr_type_list,d_sr_type_list,surf->nsr); + sr_idx_resize(k_sr_map,d_sr_map,surf->nsr); + sr_buf_resize(k_sr_global,d_sr_global,surf->nsr); + sr_buf_resize(k_sr_prob,d_sr_prob,surf->nsr); + sr_buf_resize(k_sr_adsorb,d_sr_adsorb,surf->nsr); +#endif if (surf->nsr > 0) { - int nglob,nprob; - nglob = nprob = 0; + int nglob,nprob,nadsorb; + nglob = nprob = nadsorb = 0; for (int n = 0; n < surf->nsr; n++) { if (!surf->sr[n]->kokkosable) error->all(FLERR,"Must use Kokkos-enabled surface reaction method with Kokkos"); if (strcmp(surf->sr[n]->style,"global") == 0) { +#ifdef SPARTA_KOKKOS_FIXED_LISTS if (nglob >= KOKKOS_MAX_SURF_REACT_PER_TYPE) error->all(FLERR,"Kokkos currently supports two instances of each surface reaction method"); sr_kk_global_copy[nglob].copy((SurfReactGlobalKokkos*)(surf->sr[n])); - sr_kk_global_copy[nglob].obj.pre_react(); - sr_type_list[n] = 0; - sr_map[n] = nglob; +#else + sr_buf_blit(k_sr_global,nglob,(SurfReactGlobalKokkos*)(surf->sr[n])); +#endif + KK_SR_H_GLOBAL(nglob).pre_react(); + KK_SR_H_TYPE(n) = 0; + KK_SR_H_MAP(n) = nglob; nglob++; } else if (strcmp(surf->sr[n]->style,"prob") == 0) { +#ifdef SPARTA_KOKKOS_FIXED_LISTS if (nprob >= KOKKOS_MAX_SURF_REACT_PER_TYPE) error->all(FLERR,"Kokkos currently supports two instances of each surface reaction method"); sr_kk_prob_copy[nprob].copy((SurfReactProbKokkos*)(surf->sr[n])); - sr_kk_prob_copy[nprob].obj.pre_react(); - sr_type_list[n] = 1; - sr_map[n] = nprob; +#else + sr_buf_blit(k_sr_prob,nprob,(SurfReactProbKokkos*)(surf->sr[n])); +#endif + KK_SR_H_PROB(nprob).pre_react(); + KK_SR_H_TYPE(n) = 1; + KK_SR_H_MAP(n) = nprob; nprob++; + } else if (strcmp(surf->sr[n]->style,"adsorb") == 0) { +#ifdef SPARTA_KOKKOS_FIXED_LISTS + if (nadsorb >= KOKKOS_MAX_SURF_REACT_PER_TYPE) + error->all(FLERR,"Kokkos currently supports two instances of each surface reaction method"); + sr_kk_adsorb_copy[nadsorb].copy((SurfReactAdsorbKokkos*)(surf->sr[n])); +#else + sr_buf_blit(k_sr_adsorb,nadsorb,(SurfReactAdsorbKokkos*)(surf->sr[n])); +#endif + KK_SR_H_ADSORB(nadsorb).pre_react(); + KK_SR_H_TYPE(n) = 2; + KK_SR_H_MAP(n) = nadsorb; + nadsorb++; } else { - error->all(FLERR,"This Kokkos surf_collide style supports only surf_react global/prob; surf_react adsorb requires surf_collide cll"); + error->all(FLERR,"Unknown Kokkos surface reaction method"); } } + +#ifndef SPARTA_KOKKOS_FIXED_LISTS + + // the models were blitted into the host image of the buffers and their + // pre_react() ran there; push the result to the device + + sr_buf_sync(k_sr_global,d_sr_global); + sr_buf_sync(k_sr_prob,d_sr_prob); + sr_buf_sync(k_sr_adsorb,d_sr_adsorb); + sr_idx_sync(k_sr_type_list,d_sr_type_list); + sr_idx_sync(k_sr_map,d_sr_map); +#endif } if (random == NULL) { @@ -315,8 +367,9 @@ void SurfCollideDiffuseKokkos::post_collide() // would be orphaned and its allocation never freed for (int n = 0; n < surf->nsr; n++) { - if (sr_type_list[n] == 0) sr_kk_global_copy[sr_map[n]].obj.post_react(); - else sr_kk_prob_copy[sr_map[n]].obj.post_react(); + if (KK_SR_H_TYPE(n) == 0) KK_SR_H_GLOBAL(KK_SR_H_MAP(n)).post_react(); + else if (KK_SR_H_TYPE(n) == 1) KK_SR_H_PROB(KK_SR_H_MAP(n)).post_react(); + else KK_SR_H_ADSORB(KK_SR_H_MAP(n)).post_react(); } } @@ -344,17 +397,31 @@ void SurfCollideDiffuseKokkos::backup() } if (surf->nsr > 0) { - int nglob,nprob; - nglob = nprob = 0; + int nglob,nprob,nadsorb; + nglob = nprob = nadsorb = 0; for (int n = 0; n < surf->nsr; n++) { if (strcmp(surf->sr[n]->style,"global") == 0) { - sr_kk_global_copy[nglob].obj.backup(); + KK_SR_H_GLOBAL(nglob).backup(); nglob++; } else if (strcmp(surf->sr[n]->style,"prob") == 0) { - sr_kk_prob_copy[nprob].obj.backup(); + KK_SR_H_PROB(nprob).backup(); nprob++; + } else if (strcmp(surf->sr[n]->style,"adsorb") == 0) { + KK_SR_H_ADSORB(nadsorb).backup(); + nadsorb++; } } + +#ifndef SPARTA_KOKKOS_FIXED_LISTS + + // backup() rewrites members of each model -- d_particles above all, which + // a grow reallocates -- so the device image of the buffers is stale + // until it is pushed again + + sr_buf_sync(k_sr_global,d_sr_global); + sr_buf_sync(k_sr_prob,d_sr_prob); + sr_buf_sync(k_sr_adsorb,d_sr_adsorb); +#endif } #ifdef SPARTA_KOKKOS_EXACT @@ -369,17 +436,31 @@ void SurfCollideDiffuseKokkos::backup() void SurfCollideDiffuseKokkos::restore() { if (surf->nsr > 0) { - int nglob,nprob; - nglob = nprob = 0; + int nglob,nprob,nadsorb; + nglob = nprob = nadsorb = 0; for (int n = 0; n < surf->nsr; n++) { if (strcmp(surf->sr[n]->style,"global") == 0) { - sr_kk_global_copy[nglob].obj.restore(); + KK_SR_H_GLOBAL(nglob).restore(); nglob++; } else if (strcmp(surf->sr[n]->style,"prob") == 0) { - sr_kk_prob_copy[nprob].obj.restore(); + KK_SR_H_PROB(nprob).restore(); nprob++; + } else if (strcmp(surf->sr[n]->style,"adsorb") == 0) { + KK_SR_H_ADSORB(nadsorb).restore(); + nadsorb++; } } + +#ifndef SPARTA_KOKKOS_FIXED_LISTS + + // restore() writes no member of a model today, only deep_copies into Views + // the model already holds, but the buffers are pushed for the same reason + // backup() pushes them: the device image must never trail the host one + + sr_buf_sync(k_sr_global,d_sr_global); + sr_buf_sync(k_sr_prob,d_sr_prob); + sr_buf_sync(k_sr_adsorb,d_sr_adsorb); +#endif } Kokkos::deep_copy(d_scalars,0); diff --git a/src/KOKKOS/surf_collide_diffuse_kokkos.h b/src/KOKKOS/surf_collide_diffuse_kokkos.h index 4cdcd88df..8ef141809 100644 --- a/src/KOKKOS/surf_collide_diffuse_kokkos.h +++ b/src/KOKKOS/surf_collide_diffuse_kokkos.h @@ -31,6 +31,7 @@ SurfCollideStyle(diffuse/kk,SurfCollideDiffuseKokkos) #include "fix_vibmode_kokkos.h" #include "surf_react_global_kokkos.h" #include "surf_react_prob_kokkos.h" +#include "surf_react_adsorb_kokkos.h" namespace SPARTA_NS { @@ -92,10 +93,23 @@ class SurfCollideDiffuseKokkos : public SurfCollideDiffuse { KKCopy fix_ambi_kk_copy; KKCopy fix_vibmode_kk_copy; + // the active surf react models this model may dispatch to, partitioned by + // style. Two representations, selected by SPARTA_KOKKOS_FIXED_LISTS (see + // kokkos_type.h); the device dispatch site in collide_kokkos() below is + // written once, against the KK_SR_* accessors defined there. + +#ifdef SPARTA_KOKKOS_FIXED_LISTS int sr_type_list[KOKKOS_MAX_TOT_SURF_REACT]; int sr_map[KOKKOS_MAX_TOT_SURF_REACT]; KKCopy sr_kk_global_copy[KOKKOS_MAX_SURF_REACT_PER_TYPE]; KKCopy sr_kk_prob_copy[KOKKOS_MAX_SURF_REACT_PER_TYPE]; + KKCopy sr_kk_adsorb_copy[KOKKOS_MAX_SURF_REACT_PER_TYPE]; +#else + DAT::tdual_int_1d k_sr_type_list,k_sr_map; + DAT::t_int_1d d_sr_type_list,d_sr_map; + DAT::tdual_char_1d k_sr_global,k_sr_prob,k_sr_adsorb; + DAT::t_char_1d d_sr_global,d_sr_prob,d_sr_adsorb; +#endif public: @@ -131,17 +145,27 @@ class SurfCollideDiffuseKokkos : public SurfCollideDiffuse { reaction = 0; int velreset = 0; - if (REACT) { + // isr < 0 means this surface element or box face has no reaction model, + // even though this surf collide instance is used somewhere that does. + // REACT is a compile-time flag for the whole kernel, so the runtime test + // is still needed; the host makes the same one + // (surf_collide_diffuse.cpp:146 "if (isr >= 0)"). Without it, + // KK_SR_TYPE(-1) reads out of bounds and dispatches on garbage + + if (REACT && isr >= 0) { if (ambi_flag || vibmode_flag) memcpy(&iorig,ip,sizeof(Particle::OnePart)); - int sr_type = sr_type_list[isr]; - int m = sr_map[isr]; + int sr_type = KK_SR_TYPE(isr); + int m = KK_SR_MAP(isr); if (sr_type == 0) { - reaction = sr_kk_global_copy[m].obj. + reaction = KK_SR_GLOBAL(m). react_kokkos(ip,isurf,norm,jp,velreset,d_retry,d_nlocal); } else if (sr_type == 1) { - reaction = sr_kk_prob_copy[m].obj. + reaction = KK_SR_PROB(m). + react_kokkos(ip,isurf,norm,jp,velreset,d_retry,d_nlocal); + } else if (sr_type == 2) { + reaction = KK_SR_ADSORB(m). react_kokkos(ip,isurf,norm,jp,velreset,d_retry,d_nlocal); } diff --git a/src/KOKKOS/surf_collide_impulsive_kokkos.cpp b/src/KOKKOS/surf_collide_impulsive_kokkos.cpp index db4ffb185..b1d8c7883 100644 --- a/src/KOKKOS/surf_collide_impulsive_kokkos.cpp +++ b/src/KOKKOS/surf_collide_impulsive_kokkos.cpp @@ -49,8 +49,11 @@ SurfCollideImpulsiveKokkos::SurfCollideImpulsiveKokkos(SPARTA *sparta, int narg, SurfCollideImpulsive(sparta, narg, arg), fix_ambi_kk_copy(sparta), fix_vibmode_kk_copy(sparta), +#ifdef SPARTA_KOKKOS_FIXED_LISTS sr_kk_global_copy{VAL_2(KKCopy(sparta))}, sr_kk_prob_copy{VAL_2(KKCopy(sparta))}, + sr_kk_adsorb_copy{VAL_2(KKCopy(sparta))}, +#endif rand_pool(12345 + comm->me #ifdef SPARTA_KOKKOS_EXACT , sparta @@ -86,8 +89,11 @@ SurfCollideImpulsiveKokkos::SurfCollideImpulsiveKokkos(SPARTA *sparta) : SurfCollideImpulsive(sparta), fix_ambi_kk_copy(sparta), fix_vibmode_kk_copy(sparta), +#ifdef SPARTA_KOKKOS_FIXED_LISTS sr_kk_global_copy{VAL_2(KKCopy(sparta))}, sr_kk_prob_copy{VAL_2(KKCopy(sparta))}, + sr_kk_adsorb_copy{VAL_2(KKCopy(sparta))}, +#endif rand_pool(12345 // seed doesn't matter since it will just be copied over #ifdef SPARTA_KOKKOS_EXACT , sparta @@ -236,36 +242,82 @@ void SurfCollideImpulsiveKokkos::pre_collide() fix_vibmode_kk_copy.copy(vfix_kk); } +#ifdef SPARTA_KOKKOS_FIXED_LISTS if (surf->nsr > KOKKOS_MAX_TOT_SURF_REACT) error->all(FLERR,"Kokkos currently supports two instances of each surface reaction method"); +#else + + // the buffers must be sized before anything is blitted into them. surf->nsr + // bounds the count of every individual style, so sizing all of them to it + // needs no counting pass, and the loop below still runs pre_react() in + // surf react list order + + sr_idx_resize(k_sr_type_list,d_sr_type_list,surf->nsr); + sr_idx_resize(k_sr_map,d_sr_map,surf->nsr); + sr_buf_resize(k_sr_global,d_sr_global,surf->nsr); + sr_buf_resize(k_sr_prob,d_sr_prob,surf->nsr); + sr_buf_resize(k_sr_adsorb,d_sr_adsorb,surf->nsr); +#endif if (surf->nsr > 0) { - int nglob,nprob; - nglob = nprob = 0; + int nglob,nprob,nadsorb; + nglob = nprob = nadsorb = 0; for (int n = 0; n < surf->nsr; n++) { if (!surf->sr[n]->kokkosable) error->all(FLERR,"Must use Kokkos-enabled surface reaction method with Kokkos"); if (strcmp(surf->sr[n]->style,"global") == 0) { +#ifdef SPARTA_KOKKOS_FIXED_LISTS if (nglob >= KOKKOS_MAX_SURF_REACT_PER_TYPE) error->all(FLERR,"Kokkos currently supports two instances of each surface reaction method"); sr_kk_global_copy[nglob].copy((SurfReactGlobalKokkos*)(surf->sr[n])); - sr_kk_global_copy[nglob].obj.pre_react(); - sr_type_list[n] = 0; - sr_map[n] = nglob; +#else + sr_buf_blit(k_sr_global,nglob,(SurfReactGlobalKokkos*)(surf->sr[n])); +#endif + KK_SR_H_GLOBAL(nglob).pre_react(); + KK_SR_H_TYPE(n) = 0; + KK_SR_H_MAP(n) = nglob; nglob++; } else if (strcmp(surf->sr[n]->style,"prob") == 0) { +#ifdef SPARTA_KOKKOS_FIXED_LISTS if (nprob >= KOKKOS_MAX_SURF_REACT_PER_TYPE) error->all(FLERR,"Kokkos currently supports two instances of each surface reaction method"); sr_kk_prob_copy[nprob].copy((SurfReactProbKokkos*)(surf->sr[n])); - sr_kk_prob_copy[nprob].obj.pre_react(); - sr_type_list[n] = 1; - sr_map[n] = nprob; +#else + sr_buf_blit(k_sr_prob,nprob,(SurfReactProbKokkos*)(surf->sr[n])); +#endif + KK_SR_H_PROB(nprob).pre_react(); + KK_SR_H_TYPE(n) = 1; + KK_SR_H_MAP(n) = nprob; nprob++; + } else if (strcmp(surf->sr[n]->style,"adsorb") == 0) { +#ifdef SPARTA_KOKKOS_FIXED_LISTS + if (nadsorb >= KOKKOS_MAX_SURF_REACT_PER_TYPE) + error->all(FLERR,"Kokkos currently supports two instances of each surface reaction method"); + sr_kk_adsorb_copy[nadsorb].copy((SurfReactAdsorbKokkos*)(surf->sr[n])); +#else + sr_buf_blit(k_sr_adsorb,nadsorb,(SurfReactAdsorbKokkos*)(surf->sr[n])); +#endif + KK_SR_H_ADSORB(nadsorb).pre_react(); + KK_SR_H_TYPE(n) = 2; + KK_SR_H_MAP(n) = nadsorb; + nadsorb++; } else { - error->all(FLERR,"This Kokkos surf_collide style supports only surf_react global/prob; surf_react adsorb requires surf_collide cll"); + error->all(FLERR,"Unknown Kokkos surface reaction method"); } } +#ifndef SPARTA_KOKKOS_FIXED_LISTS + + // the models were blitted into the host image of the buffers and their + // pre_react() ran there; push the result to the device + + sr_buf_sync(k_sr_global,d_sr_global); + sr_buf_sync(k_sr_prob,d_sr_prob); + sr_buf_sync(k_sr_adsorb,d_sr_adsorb); + sr_idx_sync(k_sr_type_list,d_sr_type_list); + sr_idx_sync(k_sr_map,d_sr_map); +#endif + } if (random == NULL) { @@ -317,8 +369,9 @@ void SurfCollideImpulsiveKokkos::post_collide() // would be orphaned and its allocation never freed for (int n = 0; n < surf->nsr; n++) { - if (sr_type_list[n] == 0) sr_kk_global_copy[sr_map[n]].obj.post_react(); - else sr_kk_prob_copy[sr_map[n]].obj.post_react(); + if (KK_SR_H_TYPE(n) == 0) KK_SR_H_GLOBAL(KK_SR_H_MAP(n)).post_react(); + else if (KK_SR_H_TYPE(n) == 1) KK_SR_H_PROB(KK_SR_H_MAP(n)).post_react(); + else KK_SR_H_ADSORB(KK_SR_H_MAP(n)).post_react(); } } @@ -330,17 +383,31 @@ void SurfCollideImpulsiveKokkos::backup() d_particles = particle_kk->k_particles.view_device(); if (surf->nsr > 0) { - int nglob,nprob; - nglob = nprob = 0; + int nglob,nprob,nadsorb; + nglob = nprob = nadsorb = 0; for (int n = 0; n < surf->nsr; n++) { if (strcmp(surf->sr[n]->style,"global") == 0) { - sr_kk_global_copy[nglob].obj.backup(); + KK_SR_H_GLOBAL(nglob).backup(); nglob++; } else if (strcmp(surf->sr[n]->style,"prob") == 0) { - sr_kk_prob_copy[nprob].obj.backup(); + KK_SR_H_PROB(nprob).backup(); nprob++; + } else if (strcmp(surf->sr[n]->style,"adsorb") == 0) { + KK_SR_H_ADSORB(nadsorb).backup(); + nadsorb++; } } + +#ifndef SPARTA_KOKKOS_FIXED_LISTS + + // backup() rewrites members of each model -- d_particles above all, which + // a grow reallocates -- so the device image of the buffers is stale + // until it is pushed again + + sr_buf_sync(k_sr_global,d_sr_global); + sr_buf_sync(k_sr_prob,d_sr_prob); + sr_buf_sync(k_sr_adsorb,d_sr_adsorb); +#endif } #ifdef SPARTA_KOKKOS_EXACT @@ -355,17 +422,31 @@ void SurfCollideImpulsiveKokkos::backup() void SurfCollideImpulsiveKokkos::restore() { if (surf->nsr > 0) { - int nglob,nprob; - nglob = nprob = 0; + int nglob,nprob,nadsorb; + nglob = nprob = nadsorb = 0; for (int n = 0; n < surf->nsr; n++) { if (strcmp(surf->sr[n]->style,"global") == 0) { - sr_kk_global_copy[nglob].obj.restore(); + KK_SR_H_GLOBAL(nglob).restore(); nglob++; } else if (strcmp(surf->sr[n]->style,"prob") == 0) { - sr_kk_prob_copy[nprob].obj.restore(); + KK_SR_H_PROB(nprob).restore(); nprob++; + } else if (strcmp(surf->sr[n]->style,"adsorb") == 0) { + KK_SR_H_ADSORB(nadsorb).restore(); + nadsorb++; } } + +#ifndef SPARTA_KOKKOS_FIXED_LISTS + + // restore() writes no member of a model today, only deep_copies into Views + // the model already holds, but the buffers are pushed for the same reason + // backup() pushes them: the device image must never trail the host one + + sr_buf_sync(k_sr_global,d_sr_global); + sr_buf_sync(k_sr_prob,d_sr_prob); + sr_buf_sync(k_sr_adsorb,d_sr_adsorb); +#endif } Kokkos::deep_copy(d_scalars,0); diff --git a/src/KOKKOS/surf_collide_impulsive_kokkos.h b/src/KOKKOS/surf_collide_impulsive_kokkos.h index 7e47ad83a..8743e05d7 100644 --- a/src/KOKKOS/surf_collide_impulsive_kokkos.h +++ b/src/KOKKOS/surf_collide_impulsive_kokkos.h @@ -31,6 +31,7 @@ SurfCollideStyle(impulsive/kk,SurfCollideImpulsiveKokkos) #include "fix_vibmode_kokkos.h" #include "surf_react_global_kokkos.h" #include "surf_react_prob_kokkos.h" +#include "surf_react_adsorb_kokkos.h" namespace SPARTA_NS { @@ -90,10 +91,23 @@ class SurfCollideImpulsiveKokkos : public SurfCollideImpulsive { KKCopy fix_ambi_kk_copy; KKCopy fix_vibmode_kk_copy; + // the active surf react models this model may dispatch to, partitioned by + // style. Two representations, selected by SPARTA_KOKKOS_FIXED_LISTS (see + // kokkos_type.h); the device dispatch site in collide_kokkos() below is + // written once, against the KK_SR_* accessors defined there. + +#ifdef SPARTA_KOKKOS_FIXED_LISTS int sr_type_list[KOKKOS_MAX_TOT_SURF_REACT]; int sr_map[KOKKOS_MAX_TOT_SURF_REACT]; KKCopy sr_kk_global_copy[KOKKOS_MAX_SURF_REACT_PER_TYPE]; KKCopy sr_kk_prob_copy[KOKKOS_MAX_SURF_REACT_PER_TYPE]; + KKCopy sr_kk_adsorb_copy[KOKKOS_MAX_SURF_REACT_PER_TYPE]; +#else + DAT::tdual_int_1d k_sr_type_list,k_sr_map; + DAT::t_int_1d d_sr_type_list,d_sr_map; + DAT::tdual_char_1d k_sr_global,k_sr_prob,k_sr_adsorb; + DAT::t_char_1d d_sr_global,d_sr_prob,d_sr_adsorb; +#endif public: @@ -132,14 +146,17 @@ class SurfCollideImpulsiveKokkos : public SurfCollideImpulsive { if (REACT && isr >= 0) { if (ambi_flag || vibmode_flag) memcpy(&iorig,ip,sizeof(Particle::OnePart)); - int sr_type = sr_type_list[isr]; - int m = sr_map[isr]; + int sr_type = KK_SR_TYPE(isr); + int m = KK_SR_MAP(isr); if (sr_type == 0) { - reaction = sr_kk_global_copy[m].obj. + reaction = KK_SR_GLOBAL(m). react_kokkos(ip,isurf,norm,jp,velreset,d_retry,d_nlocal); } else if (sr_type == 1) { - reaction = sr_kk_prob_copy[m].obj. + reaction = KK_SR_PROB(m). + react_kokkos(ip,isurf,norm,jp,velreset,d_retry,d_nlocal); + } else if (sr_type == 2) { + reaction = KK_SR_ADSORB(m). react_kokkos(ip,isurf,norm,jp,velreset,d_retry,d_nlocal); } diff --git a/src/KOKKOS/surf_collide_piston_kokkos.cpp b/src/KOKKOS/surf_collide_piston_kokkos.cpp index ee6016f9e..2a0cdda41 100644 --- a/src/KOKKOS/surf_collide_piston_kokkos.cpp +++ b/src/KOKKOS/surf_collide_piston_kokkos.cpp @@ -33,9 +33,12 @@ using namespace SPARTA_NS; SurfCollidePistonKokkos::SurfCollidePistonKokkos(SPARTA *sparta, int narg, char **arg) : SurfCollidePiston(sparta, narg, arg), fix_ambi_kk_copy(sparta), - fix_vibmode_kk_copy(sparta), - sr_kk_global_copy{VAL_2(KKCopy(sparta))}, - sr_kk_prob_copy{VAL_2(KKCopy(sparta))} + fix_vibmode_kk_copy(sparta) +#ifdef SPARTA_KOKKOS_FIXED_LISTS + , sr_kk_global_copy{VAL_2(KKCopy(sparta))} + , sr_kk_prob_copy{VAL_2(KKCopy(sparta))} + , sr_kk_adsorb_copy{VAL_2(KKCopy(sparta))} +#endif { kokkosable = 1; @@ -55,9 +58,12 @@ SurfCollidePistonKokkos::SurfCollidePistonKokkos(SPARTA *sparta, int narg, char SurfCollidePistonKokkos::SurfCollidePistonKokkos(SPARTA *sparta) : SurfCollidePiston(sparta), fix_ambi_kk_copy(sparta), - fix_vibmode_kk_copy(sparta), - sr_kk_global_copy{VAL_2(KKCopy(sparta))}, - sr_kk_prob_copy{VAL_2(KKCopy(sparta))} + fix_vibmode_kk_copy(sparta) +#ifdef SPARTA_KOKKOS_FIXED_LISTS + , sr_kk_global_copy{VAL_2(KKCopy(sparta))} + , sr_kk_prob_copy{VAL_2(KKCopy(sparta))} + , sr_kk_adsorb_copy{VAL_2(KKCopy(sparta))} +#endif { copy = 1; } @@ -107,35 +113,81 @@ void SurfCollidePistonKokkos::pre_collide() fix_vibmode_kk_copy.copy(vfix_kk); } +#ifdef SPARTA_KOKKOS_FIXED_LISTS if (surf->nsr > KOKKOS_MAX_TOT_SURF_REACT) error->all(FLERR,"Kokkos currently supports a limited number of surface reaction methods"); +#else + + // the buffers must be sized before anything is blitted into them. surf->nsr + // bounds the count of every individual style, so sizing all of them to it + // needs no counting pass, and the loop below still runs pre_react() in + // surf react list order + + sr_idx_resize(k_sr_type_list,d_sr_type_list,surf->nsr); + sr_idx_resize(k_sr_map,d_sr_map,surf->nsr); + sr_buf_resize(k_sr_global,d_sr_global,surf->nsr); + sr_buf_resize(k_sr_prob,d_sr_prob,surf->nsr); + sr_buf_resize(k_sr_adsorb,d_sr_adsorb,surf->nsr); +#endif if (surf->nsr > 0) { - int nglob,nprob; - nglob = nprob = 0; + int nglob,nprob,nadsorb; + nglob = nprob = nadsorb = 0; for (int n = 0; n < surf->nsr; n++) { if (!surf->sr[n]->kokkosable) error->all(FLERR,"Must use Kokkos-enabled surface reaction method with Kokkos"); if (strcmp(surf->sr[n]->style,"global") == 0) { +#ifdef SPARTA_KOKKOS_FIXED_LISTS if (nglob >= KOKKOS_MAX_SURF_REACT_PER_TYPE) error->all(FLERR,"Kokkos currently supports two instances of each surface reaction method"); sr_kk_global_copy[nglob].copy((SurfReactGlobalKokkos*)(surf->sr[n])); - sr_kk_global_copy[nglob].obj.pre_react(); - sr_type_list[n] = 0; - sr_map[n] = nglob; +#else + sr_buf_blit(k_sr_global,nglob,(SurfReactGlobalKokkos*)(surf->sr[n])); +#endif + KK_SR_H_GLOBAL(nglob).pre_react(); + KK_SR_H_TYPE(n) = 0; + KK_SR_H_MAP(n) = nglob; nglob++; } else if (strcmp(surf->sr[n]->style,"prob") == 0) { +#ifdef SPARTA_KOKKOS_FIXED_LISTS if (nprob >= KOKKOS_MAX_SURF_REACT_PER_TYPE) error->all(FLERR,"Kokkos currently supports two instances of each surface reaction method"); sr_kk_prob_copy[nprob].copy((SurfReactProbKokkos*)(surf->sr[n])); - sr_kk_prob_copy[nprob].obj.pre_react(); - sr_type_list[n] = 1; - sr_map[n] = nprob; +#else + sr_buf_blit(k_sr_prob,nprob,(SurfReactProbKokkos*)(surf->sr[n])); +#endif + KK_SR_H_PROB(nprob).pre_react(); + KK_SR_H_TYPE(n) = 1; + KK_SR_H_MAP(n) = nprob; nprob++; + } else if (strcmp(surf->sr[n]->style,"adsorb") == 0) { +#ifdef SPARTA_KOKKOS_FIXED_LISTS + if (nadsorb >= KOKKOS_MAX_SURF_REACT_PER_TYPE) + error->all(FLERR,"Kokkos currently supports two instances of each surface reaction method"); + sr_kk_adsorb_copy[nadsorb].copy((SurfReactAdsorbKokkos*)(surf->sr[n])); +#else + sr_buf_blit(k_sr_adsorb,nadsorb,(SurfReactAdsorbKokkos*)(surf->sr[n])); +#endif + KK_SR_H_ADSORB(nadsorb).pre_react(); + KK_SR_H_TYPE(n) = 2; + KK_SR_H_MAP(n) = nadsorb; + nadsorb++; } else { - error->all(FLERR,"This Kokkos surf_collide style supports only surf_react global/prob; surf_react adsorb requires surf_collide cll"); + error->all(FLERR,"Unknown Kokkos surface reaction method"); } } + +#ifndef SPARTA_KOKKOS_FIXED_LISTS + + // the models were blitted into the host image of the buffers and their + // pre_react() ran there; push the result to the device + + sr_buf_sync(k_sr_global,d_sr_global); + sr_buf_sync(k_sr_prob,d_sr_prob); + sr_buf_sync(k_sr_adsorb,d_sr_adsorb); + sr_idx_sync(k_sr_type_list,d_sr_type_list); + sr_idx_sync(k_sr_map,d_sr_map); +#endif } ParticleKokkos* particle_kk = (ParticleKokkos*) particle; @@ -166,8 +218,9 @@ void SurfCollidePistonKokkos::post_collide() // would be orphaned and its allocation never freed for (int n = 0; n < surf->nsr; n++) { - if (sr_type_list[n] == 0) sr_kk_global_copy[sr_map[n]].obj.post_react(); - else sr_kk_prob_copy[sr_map[n]].obj.post_react(); + if (KK_SR_H_TYPE(n) == 0) KK_SR_H_GLOBAL(KK_SR_H_MAP(n)).post_react(); + else if (KK_SR_H_TYPE(n) == 1) KK_SR_H_PROB(KK_SR_H_MAP(n)).post_react(); + else KK_SR_H_ADSORB(KK_SR_H_MAP(n)).post_react(); } } @@ -195,17 +248,31 @@ void SurfCollidePistonKokkos::backup() } if (surf->nsr > 0) { - int nglob,nprob; - nglob = nprob = 0; + int nglob,nprob,nadsorb; + nglob = nprob = nadsorb = 0; for (int n = 0; n < surf->nsr; n++) { if (strcmp(surf->sr[n]->style,"global") == 0) { - sr_kk_global_copy[nglob].obj.backup(); + KK_SR_H_GLOBAL(nglob).backup(); nglob++; } else if (strcmp(surf->sr[n]->style,"prob") == 0) { - sr_kk_prob_copy[nprob].obj.backup(); + KK_SR_H_PROB(nprob).backup(); nprob++; + } else if (strcmp(surf->sr[n]->style,"adsorb") == 0) { + KK_SR_H_ADSORB(nadsorb).backup(); + nadsorb++; } } + +#ifndef SPARTA_KOKKOS_FIXED_LISTS + + // backup() rewrites members of each model -- d_particles above all, which + // a grow reallocates -- so the device image of the buffers is stale + // until it is pushed again + + sr_buf_sync(k_sr_global,d_sr_global); + sr_buf_sync(k_sr_prob,d_sr_prob); + sr_buf_sync(k_sr_adsorb,d_sr_adsorb); +#endif } } @@ -214,17 +281,31 @@ void SurfCollidePistonKokkos::backup() void SurfCollidePistonKokkos::restore() { if (surf->nsr > 0) { - int nglob,nprob; - nglob = nprob = 0; + int nglob,nprob,nadsorb; + nglob = nprob = nadsorb = 0; for (int n = 0; n < surf->nsr; n++) { if (strcmp(surf->sr[n]->style,"global") == 0) { - sr_kk_global_copy[nglob].obj.restore(); + KK_SR_H_GLOBAL(nglob).restore(); nglob++; } else if (strcmp(surf->sr[n]->style,"prob") == 0) { - sr_kk_prob_copy[nprob].obj.restore(); + KK_SR_H_PROB(nprob).restore(); nprob++; + } else if (strcmp(surf->sr[n]->style,"adsorb") == 0) { + KK_SR_H_ADSORB(nadsorb).restore(); + nadsorb++; } } + +#ifndef SPARTA_KOKKOS_FIXED_LISTS + + // restore() writes no member of a model today, only deep_copies into Views + // the model already holds, but the buffers are pushed for the same reason + // backup() pushes them: the device image must never trail the host one + + sr_buf_sync(k_sr_global,d_sr_global); + sr_buf_sync(k_sr_prob,d_sr_prob); + sr_buf_sync(k_sr_adsorb,d_sr_adsorb); +#endif } Kokkos::deep_copy(d_scalars,0); diff --git a/src/KOKKOS/surf_collide_piston_kokkos.h b/src/KOKKOS/surf_collide_piston_kokkos.h index 4ae29702c..869e61a3c 100644 --- a/src/KOKKOS/surf_collide_piston_kokkos.h +++ b/src/KOKKOS/surf_collide_piston_kokkos.h @@ -31,6 +31,7 @@ SurfCollideStyle(piston/kk,SurfCollidePistonKokkos) #include "fix_vibmode_kokkos.h" #include "surf_react_global_kokkos.h" #include "surf_react_prob_kokkos.h" +#include "surf_react_adsorb_kokkos.h" namespace SPARTA_NS { @@ -72,10 +73,23 @@ class SurfCollidePistonKokkos : public SurfCollidePiston { KKCopy fix_ambi_kk_copy; KKCopy fix_vibmode_kk_copy; + // the active surf react models this model may dispatch to, partitioned by + // style. Two representations, selected by SPARTA_KOKKOS_FIXED_LISTS (see + // kokkos_type.h); the device dispatch site in collide_kokkos() below is + // written once, against the KK_SR_* accessors defined there. + +#ifdef SPARTA_KOKKOS_FIXED_LISTS int sr_type_list[KOKKOS_MAX_TOT_SURF_REACT]; int sr_map[KOKKOS_MAX_TOT_SURF_REACT]; KKCopy sr_kk_global_copy[KOKKOS_MAX_SURF_REACT_PER_TYPE]; KKCopy sr_kk_prob_copy[KOKKOS_MAX_SURF_REACT_PER_TYPE]; + KKCopy sr_kk_adsorb_copy[KOKKOS_MAX_SURF_REACT_PER_TYPE]; +#else + DAT::tdual_int_1d k_sr_type_list,k_sr_map; + DAT::t_int_1d d_sr_type_list,d_sr_map; + DAT::tdual_char_1d k_sr_global,k_sr_prob,k_sr_adsorb; + DAT::t_char_1d d_sr_global,d_sr_prob,d_sr_adsorb; +#endif public: @@ -111,17 +125,27 @@ class SurfCollidePistonKokkos : public SurfCollidePiston { reaction = 0; int velreset = 0; - if (REACT) { + // isr < 0 means this surface element or box face has no reaction model, + // even though this surf collide instance is used somewhere that does. + // REACT is a compile-time flag for the whole kernel, so the runtime test + // is still needed; the host makes the same one + // (surf_collide_diffuse.cpp:146 "if (isr >= 0)"). Without it, + // KK_SR_TYPE(-1) reads out of bounds and dispatches on garbage + + if (REACT && isr >= 0) { if (ambi_flag || vibmode_flag) memcpy(&iorig,ip,sizeof(Particle::OnePart)); - int sr_type = sr_type_list[isr]; - int m = sr_map[isr]; + int sr_type = KK_SR_TYPE(isr); + int m = KK_SR_MAP(isr); if (sr_type == 0) { - reaction = sr_kk_global_copy[m].obj. + reaction = KK_SR_GLOBAL(m). react_kokkos(ip,isurf,norm,jp,velreset,d_retry,d_nlocal); } else if (sr_type == 1) { - reaction = sr_kk_prob_copy[m].obj. + reaction = KK_SR_PROB(m). + react_kokkos(ip,isurf,norm,jp,velreset,d_retry,d_nlocal); + } else if (sr_type == 2) { + reaction = KK_SR_ADSORB(m). react_kokkos(ip,isurf,norm,jp,velreset,d_retry,d_nlocal); } diff --git a/src/KOKKOS/surf_collide_specular_kokkos.cpp b/src/KOKKOS/surf_collide_specular_kokkos.cpp index 592d5d37f..691a82fee 100644 --- a/src/KOKKOS/surf_collide_specular_kokkos.cpp +++ b/src/KOKKOS/surf_collide_specular_kokkos.cpp @@ -29,9 +29,12 @@ using namespace SPARTA_NS; SurfCollideSpecularKokkos::SurfCollideSpecularKokkos(SPARTA *sparta, int narg, char **arg) : SurfCollideSpecular(sparta, narg, arg), fix_ambi_kk_copy(sparta), - fix_vibmode_kk_copy(sparta), - sr_kk_global_copy{VAL_2(KKCopy(sparta))}, - sr_kk_prob_copy{VAL_2(KKCopy(sparta))} + fix_vibmode_kk_copy(sparta) +#ifdef SPARTA_KOKKOS_FIXED_LISTS + , sr_kk_global_copy{VAL_2(KKCopy(sparta))} + , sr_kk_prob_copy{VAL_2(KKCopy(sparta))} + , sr_kk_adsorb_copy{VAL_2(KKCopy(sparta))} +#endif { kokkosable = 1; @@ -51,9 +54,12 @@ SurfCollideSpecularKokkos::SurfCollideSpecularKokkos(SPARTA *sparta, int narg, c SurfCollideSpecularKokkos::SurfCollideSpecularKokkos(SPARTA *sparta) : SurfCollideSpecular(sparta), fix_ambi_kk_copy(sparta), - fix_vibmode_kk_copy(sparta), - sr_kk_global_copy{VAL_2(KKCopy(sparta))}, - sr_kk_prob_copy{VAL_2(KKCopy(sparta))} + fix_vibmode_kk_copy(sparta) +#ifdef SPARTA_KOKKOS_FIXED_LISTS + , sr_kk_global_copy{VAL_2(KKCopy(sparta))} + , sr_kk_prob_copy{VAL_2(KKCopy(sparta))} + , sr_kk_adsorb_copy{VAL_2(KKCopy(sparta))} +#endif { copy = 1; } @@ -103,35 +109,81 @@ void SurfCollideSpecularKokkos::pre_collide() fix_vibmode_kk_copy.copy(vfix_kk); } +#ifdef SPARTA_KOKKOS_FIXED_LISTS if (surf->nsr > KOKKOS_MAX_TOT_SURF_REACT) error->all(FLERR,"Kokkos currently supports a limited number of surface reaction methods"); +#else + + // the buffers must be sized before anything is blitted into them. surf->nsr + // bounds the count of every individual style, so sizing all of them to it + // needs no counting pass, and the loop below still runs pre_react() in + // surf react list order + + sr_idx_resize(k_sr_type_list,d_sr_type_list,surf->nsr); + sr_idx_resize(k_sr_map,d_sr_map,surf->nsr); + sr_buf_resize(k_sr_global,d_sr_global,surf->nsr); + sr_buf_resize(k_sr_prob,d_sr_prob,surf->nsr); + sr_buf_resize(k_sr_adsorb,d_sr_adsorb,surf->nsr); +#endif if (surf->nsr > 0) { - int nglob,nprob; - nglob = nprob = 0; + int nglob,nprob,nadsorb; + nglob = nprob = nadsorb = 0; for (int n = 0; n < surf->nsr; n++) { if (!surf->sr[n]->kokkosable) error->all(FLERR,"Must use Kokkos-enabled surface reaction method with Kokkos"); if (strcmp(surf->sr[n]->style,"global") == 0) { +#ifdef SPARTA_KOKKOS_FIXED_LISTS if (nglob >= KOKKOS_MAX_SURF_REACT_PER_TYPE) error->all(FLERR,"Kokkos currently supports two instances of each surface reaction method"); sr_kk_global_copy[nglob].copy((SurfReactGlobalKokkos*)(surf->sr[n])); - sr_kk_global_copy[nglob].obj.pre_react(); - sr_type_list[n] = 0; - sr_map[n] = nglob; +#else + sr_buf_blit(k_sr_global,nglob,(SurfReactGlobalKokkos*)(surf->sr[n])); +#endif + KK_SR_H_GLOBAL(nglob).pre_react(); + KK_SR_H_TYPE(n) = 0; + KK_SR_H_MAP(n) = nglob; nglob++; } else if (strcmp(surf->sr[n]->style,"prob") == 0) { +#ifdef SPARTA_KOKKOS_FIXED_LISTS if (nprob >= KOKKOS_MAX_SURF_REACT_PER_TYPE) error->all(FLERR,"Kokkos currently supports two instances of each surface reaction method"); sr_kk_prob_copy[nprob].copy((SurfReactProbKokkos*)(surf->sr[n])); - sr_kk_prob_copy[nprob].obj.pre_react(); - sr_type_list[n] = 1; - sr_map[n] = nprob; +#else + sr_buf_blit(k_sr_prob,nprob,(SurfReactProbKokkos*)(surf->sr[n])); +#endif + KK_SR_H_PROB(nprob).pre_react(); + KK_SR_H_TYPE(n) = 1; + KK_SR_H_MAP(n) = nprob; nprob++; + } else if (strcmp(surf->sr[n]->style,"adsorb") == 0) { +#ifdef SPARTA_KOKKOS_FIXED_LISTS + if (nadsorb >= KOKKOS_MAX_SURF_REACT_PER_TYPE) + error->all(FLERR,"Kokkos currently supports two instances of each surface reaction method"); + sr_kk_adsorb_copy[nadsorb].copy((SurfReactAdsorbKokkos*)(surf->sr[n])); +#else + sr_buf_blit(k_sr_adsorb,nadsorb,(SurfReactAdsorbKokkos*)(surf->sr[n])); +#endif + KK_SR_H_ADSORB(nadsorb).pre_react(); + KK_SR_H_TYPE(n) = 2; + KK_SR_H_MAP(n) = nadsorb; + nadsorb++; } else { - error->all(FLERR,"This Kokkos surf_collide style supports only surf_react global/prob; surf_react adsorb requires surf_collide cll"); + error->all(FLERR,"Unknown Kokkos surface reaction method"); } } + +#ifndef SPARTA_KOKKOS_FIXED_LISTS + + // the models were blitted into the host image of the buffers and their + // pre_react() ran there; push the result to the device + + sr_buf_sync(k_sr_global,d_sr_global); + sr_buf_sync(k_sr_prob,d_sr_prob); + sr_buf_sync(k_sr_adsorb,d_sr_adsorb); + sr_idx_sync(k_sr_type_list,d_sr_type_list); + sr_idx_sync(k_sr_map,d_sr_map); +#endif } ParticleKokkos* particle_kk = (ParticleKokkos*) particle; @@ -162,8 +214,9 @@ void SurfCollideSpecularKokkos::post_collide() // would be orphaned and its allocation never freed for (int n = 0; n < surf->nsr; n++) { - if (sr_type_list[n] == 0) sr_kk_global_copy[sr_map[n]].obj.post_react(); - else sr_kk_prob_copy[sr_map[n]].obj.post_react(); + if (KK_SR_H_TYPE(n) == 0) KK_SR_H_GLOBAL(KK_SR_H_MAP(n)).post_react(); + else if (KK_SR_H_TYPE(n) == 1) KK_SR_H_PROB(KK_SR_H_MAP(n)).post_react(); + else KK_SR_H_ADSORB(KK_SR_H_MAP(n)).post_react(); } } @@ -191,17 +244,31 @@ void SurfCollideSpecularKokkos::backup() } if (surf->nsr > 0) { - int nglob,nprob; - nglob = nprob = 0; + int nglob,nprob,nadsorb; + nglob = nprob = nadsorb = 0; for (int n = 0; n < surf->nsr; n++) { if (strcmp(surf->sr[n]->style,"global") == 0) { - sr_kk_global_copy[nglob].obj.backup(); + KK_SR_H_GLOBAL(nglob).backup(); nglob++; } else if (strcmp(surf->sr[n]->style,"prob") == 0) { - sr_kk_prob_copy[nprob].obj.backup(); + KK_SR_H_PROB(nprob).backup(); nprob++; + } else if (strcmp(surf->sr[n]->style,"adsorb") == 0) { + KK_SR_H_ADSORB(nadsorb).backup(); + nadsorb++; } } + +#ifndef SPARTA_KOKKOS_FIXED_LISTS + + // backup() rewrites members of each model -- d_particles above all, which + // a grow reallocates -- so the device image of the buffers is stale + // until it is pushed again + + sr_buf_sync(k_sr_global,d_sr_global); + sr_buf_sync(k_sr_prob,d_sr_prob); + sr_buf_sync(k_sr_adsorb,d_sr_adsorb); +#endif } } @@ -210,17 +277,31 @@ void SurfCollideSpecularKokkos::backup() void SurfCollideSpecularKokkos::restore() { if (surf->nsr > 0) { - int nglob,nprob; - nglob = nprob = 0; + int nglob,nprob,nadsorb; + nglob = nprob = nadsorb = 0; for (int n = 0; n < surf->nsr; n++) { if (strcmp(surf->sr[n]->style,"global") == 0) { - sr_kk_global_copy[nglob].obj.restore(); + KK_SR_H_GLOBAL(nglob).restore(); nglob++; } else if (strcmp(surf->sr[n]->style,"prob") == 0) { - sr_kk_prob_copy[nprob].obj.restore(); + KK_SR_H_PROB(nprob).restore(); nprob++; + } else if (strcmp(surf->sr[n]->style,"adsorb") == 0) { + KK_SR_H_ADSORB(nadsorb).restore(); + nadsorb++; } } + +#ifndef SPARTA_KOKKOS_FIXED_LISTS + + // restore() writes no member of a model today, only deep_copies into Views + // the model already holds, but the buffers are pushed for the same reason + // backup() pushes them: the device image must never trail the host one + + sr_buf_sync(k_sr_global,d_sr_global); + sr_buf_sync(k_sr_prob,d_sr_prob); + sr_buf_sync(k_sr_adsorb,d_sr_adsorb); +#endif } Kokkos::deep_copy(d_scalars,0); diff --git a/src/KOKKOS/surf_collide_specular_kokkos.h b/src/KOKKOS/surf_collide_specular_kokkos.h index 2c8878280..c741b5cd2 100644 --- a/src/KOKKOS/surf_collide_specular_kokkos.h +++ b/src/KOKKOS/surf_collide_specular_kokkos.h @@ -31,6 +31,7 @@ SurfCollideStyle(specular/kk,SurfCollideSpecularKokkos) #include "fix_vibmode_kokkos.h" #include "surf_react_global_kokkos.h" #include "surf_react_prob_kokkos.h" +#include "surf_react_adsorb_kokkos.h" namespace SPARTA_NS { @@ -72,10 +73,23 @@ class SurfCollideSpecularKokkos : public SurfCollideSpecular { KKCopy fix_ambi_kk_copy; KKCopy fix_vibmode_kk_copy; + // the active surf react models this model may dispatch to, partitioned by + // style. Two representations, selected by SPARTA_KOKKOS_FIXED_LISTS (see + // kokkos_type.h); the device dispatch site in collide_kokkos() below is + // written once, against the KK_SR_* accessors defined there. + +#ifdef SPARTA_KOKKOS_FIXED_LISTS int sr_type_list[KOKKOS_MAX_TOT_SURF_REACT]; int sr_map[KOKKOS_MAX_TOT_SURF_REACT]; KKCopy sr_kk_global_copy[KOKKOS_MAX_SURF_REACT_PER_TYPE]; KKCopy sr_kk_prob_copy[KOKKOS_MAX_SURF_REACT_PER_TYPE]; + KKCopy sr_kk_adsorb_copy[KOKKOS_MAX_SURF_REACT_PER_TYPE]; +#else + DAT::tdual_int_1d k_sr_type_list,k_sr_map; + DAT::t_int_1d d_sr_type_list,d_sr_map; + DAT::tdual_char_1d k_sr_global,k_sr_prob,k_sr_adsorb; + DAT::t_char_1d d_sr_global,d_sr_prob,d_sr_adsorb; +#endif public: @@ -111,17 +125,27 @@ class SurfCollideSpecularKokkos : public SurfCollideSpecular { reaction = 0; int velreset = 0; - if (REACT) { + // isr < 0 means this surface element or box face has no reaction model, + // even though this surf collide instance is used somewhere that does. + // REACT is a compile-time flag for the whole kernel, so the runtime test + // is still needed; the host makes the same one + // (surf_collide_diffuse.cpp:146 "if (isr >= 0)"). Without it, + // KK_SR_TYPE(-1) reads out of bounds and dispatches on garbage + + if (REACT && isr >= 0) { if (ambi_flag || vibmode_flag) memcpy(&iorig,ip,sizeof(Particle::OnePart)); - int sr_type = sr_type_list[isr]; - int m = sr_map[isr]; + int sr_type = KK_SR_TYPE(isr); + int m = KK_SR_MAP(isr); if (sr_type == 0) { - reaction = sr_kk_global_copy[m].obj. + reaction = KK_SR_GLOBAL(m). react_kokkos(ip,isurf,norm,jp,velreset,d_retry,d_nlocal); } else if (sr_type == 1) { - reaction = sr_kk_prob_copy[m].obj. + reaction = KK_SR_PROB(m). + react_kokkos(ip,isurf,norm,jp,velreset,d_retry,d_nlocal); + } else if (sr_type == 2) { + reaction = KK_SR_ADSORB(m). react_kokkos(ip,isurf,norm,jp,velreset,d_retry,d_nlocal); } diff --git a/src/KOKKOS/surf_collide_td_kokkos.cpp b/src/KOKKOS/surf_collide_td_kokkos.cpp index 645c7ced3..723f9d2c7 100644 --- a/src/KOKKOS/surf_collide_td_kokkos.cpp +++ b/src/KOKKOS/surf_collide_td_kokkos.cpp @@ -49,8 +49,11 @@ SurfCollideTDKokkos::SurfCollideTDKokkos(SPARTA *sparta, int narg, char **arg) : SurfCollideTD(sparta, narg, arg), fix_ambi_kk_copy(sparta), fix_vibmode_kk_copy(sparta), +#ifdef SPARTA_KOKKOS_FIXED_LISTS sr_kk_global_copy{VAL_2(KKCopy(sparta))}, sr_kk_prob_copy{VAL_2(KKCopy(sparta))}, + sr_kk_adsorb_copy{VAL_2(KKCopy(sparta))}, +#endif rand_pool(12345 + comm->me #ifdef SPARTA_KOKKOS_EXACT , sparta @@ -86,8 +89,11 @@ SurfCollideTDKokkos::SurfCollideTDKokkos(SPARTA *sparta) : SurfCollideTD(sparta), fix_ambi_kk_copy(sparta), fix_vibmode_kk_copy(sparta), +#ifdef SPARTA_KOKKOS_FIXED_LISTS sr_kk_global_copy{VAL_2(KKCopy(sparta))}, sr_kk_prob_copy{VAL_2(KKCopy(sparta))}, + sr_kk_adsorb_copy{VAL_2(KKCopy(sparta))}, +#endif rand_pool(12345 // seed doesn't matter since it will just be copied over #ifdef SPARTA_KOKKOS_EXACT , sparta @@ -236,36 +242,82 @@ void SurfCollideTDKokkos::pre_collide() fix_vibmode_kk_copy.copy(vfix_kk); } +#ifdef SPARTA_KOKKOS_FIXED_LISTS if (surf->nsr > KOKKOS_MAX_TOT_SURF_REACT) error->all(FLERR,"Kokkos currently supports two instances of each surface reaction method"); +#else + + // the buffers must be sized before anything is blitted into them. surf->nsr + // bounds the count of every individual style, so sizing all of them to it + // needs no counting pass, and the loop below still runs pre_react() in + // surf react list order + + sr_idx_resize(k_sr_type_list,d_sr_type_list,surf->nsr); + sr_idx_resize(k_sr_map,d_sr_map,surf->nsr); + sr_buf_resize(k_sr_global,d_sr_global,surf->nsr); + sr_buf_resize(k_sr_prob,d_sr_prob,surf->nsr); + sr_buf_resize(k_sr_adsorb,d_sr_adsorb,surf->nsr); +#endif if (surf->nsr > 0) { - int nglob,nprob; - nglob = nprob = 0; + int nglob,nprob,nadsorb; + nglob = nprob = nadsorb = 0; for (int n = 0; n < surf->nsr; n++) { if (!surf->sr[n]->kokkosable) error->all(FLERR,"Must use Kokkos-enabled surface reaction method with Kokkos"); if (strcmp(surf->sr[n]->style,"global") == 0) { +#ifdef SPARTA_KOKKOS_FIXED_LISTS if (nglob >= KOKKOS_MAX_SURF_REACT_PER_TYPE) error->all(FLERR,"Kokkos currently supports two instances of each surface reaction method"); sr_kk_global_copy[nglob].copy((SurfReactGlobalKokkos*)(surf->sr[n])); - sr_kk_global_copy[nglob].obj.pre_react(); - sr_type_list[n] = 0; - sr_map[n] = nglob; +#else + sr_buf_blit(k_sr_global,nglob,(SurfReactGlobalKokkos*)(surf->sr[n])); +#endif + KK_SR_H_GLOBAL(nglob).pre_react(); + KK_SR_H_TYPE(n) = 0; + KK_SR_H_MAP(n) = nglob; nglob++; } else if (strcmp(surf->sr[n]->style,"prob") == 0) { +#ifdef SPARTA_KOKKOS_FIXED_LISTS if (nprob >= KOKKOS_MAX_SURF_REACT_PER_TYPE) error->all(FLERR,"Kokkos currently supports two instances of each surface reaction method"); sr_kk_prob_copy[nprob].copy((SurfReactProbKokkos*)(surf->sr[n])); - sr_kk_prob_copy[nprob].obj.pre_react(); - sr_type_list[n] = 1; - sr_map[n] = nprob; +#else + sr_buf_blit(k_sr_prob,nprob,(SurfReactProbKokkos*)(surf->sr[n])); +#endif + KK_SR_H_PROB(nprob).pre_react(); + KK_SR_H_TYPE(n) = 1; + KK_SR_H_MAP(n) = nprob; nprob++; + } else if (strcmp(surf->sr[n]->style,"adsorb") == 0) { +#ifdef SPARTA_KOKKOS_FIXED_LISTS + if (nadsorb >= KOKKOS_MAX_SURF_REACT_PER_TYPE) + error->all(FLERR,"Kokkos currently supports two instances of each surface reaction method"); + sr_kk_adsorb_copy[nadsorb].copy((SurfReactAdsorbKokkos*)(surf->sr[n])); +#else + sr_buf_blit(k_sr_adsorb,nadsorb,(SurfReactAdsorbKokkos*)(surf->sr[n])); +#endif + KK_SR_H_ADSORB(nadsorb).pre_react(); + KK_SR_H_TYPE(n) = 2; + KK_SR_H_MAP(n) = nadsorb; + nadsorb++; } else { - error->all(FLERR,"This Kokkos surf_collide style supports only surf_react global/prob; surf_react adsorb requires surf_collide cll"); + error->all(FLERR,"Unknown Kokkos surface reaction method"); } } +#ifndef SPARTA_KOKKOS_FIXED_LISTS + + // the models were blitted into the host image of the buffers and their + // pre_react() ran there; push the result to the device + + sr_buf_sync(k_sr_global,d_sr_global); + sr_buf_sync(k_sr_prob,d_sr_prob); + sr_buf_sync(k_sr_adsorb,d_sr_adsorb); + sr_idx_sync(k_sr_type_list,d_sr_type_list); + sr_idx_sync(k_sr_map,d_sr_map); +#endif + } if (random == NULL) { @@ -317,8 +369,9 @@ void SurfCollideTDKokkos::post_collide() // would be orphaned and its allocation never freed for (int n = 0; n < surf->nsr; n++) { - if (sr_type_list[n] == 0) sr_kk_global_copy[sr_map[n]].obj.post_react(); - else sr_kk_prob_copy[sr_map[n]].obj.post_react(); + if (KK_SR_H_TYPE(n) == 0) KK_SR_H_GLOBAL(KK_SR_H_MAP(n)).post_react(); + else if (KK_SR_H_TYPE(n) == 1) KK_SR_H_PROB(KK_SR_H_MAP(n)).post_react(); + else KK_SR_H_ADSORB(KK_SR_H_MAP(n)).post_react(); } } @@ -330,17 +383,31 @@ void SurfCollideTDKokkos::backup() d_particles = particle_kk->k_particles.view_device(); if (surf->nsr > 0) { - int nglob,nprob; - nglob = nprob = 0; + int nglob,nprob,nadsorb; + nglob = nprob = nadsorb = 0; for (int n = 0; n < surf->nsr; n++) { if (strcmp(surf->sr[n]->style,"global") == 0) { - sr_kk_global_copy[nglob].obj.backup(); + KK_SR_H_GLOBAL(nglob).backup(); nglob++; } else if (strcmp(surf->sr[n]->style,"prob") == 0) { - sr_kk_prob_copy[nprob].obj.backup(); + KK_SR_H_PROB(nprob).backup(); nprob++; + } else if (strcmp(surf->sr[n]->style,"adsorb") == 0) { + KK_SR_H_ADSORB(nadsorb).backup(); + nadsorb++; } } + +#ifndef SPARTA_KOKKOS_FIXED_LISTS + + // backup() rewrites members of each model -- d_particles above all, which + // a grow reallocates -- so the device image of the buffers is stale + // until it is pushed again + + sr_buf_sync(k_sr_global,d_sr_global); + sr_buf_sync(k_sr_prob,d_sr_prob); + sr_buf_sync(k_sr_adsorb,d_sr_adsorb); +#endif } #ifdef SPARTA_KOKKOS_EXACT @@ -355,17 +422,31 @@ void SurfCollideTDKokkos::backup() void SurfCollideTDKokkos::restore() { if (surf->nsr > 0) { - int nglob,nprob; - nglob = nprob = 0; + int nglob,nprob,nadsorb; + nglob = nprob = nadsorb = 0; for (int n = 0; n < surf->nsr; n++) { if (strcmp(surf->sr[n]->style,"global") == 0) { - sr_kk_global_copy[nglob].obj.restore(); + KK_SR_H_GLOBAL(nglob).restore(); nglob++; } else if (strcmp(surf->sr[n]->style,"prob") == 0) { - sr_kk_prob_copy[nprob].obj.restore(); + KK_SR_H_PROB(nprob).restore(); nprob++; + } else if (strcmp(surf->sr[n]->style,"adsorb") == 0) { + KK_SR_H_ADSORB(nadsorb).restore(); + nadsorb++; } } + +#ifndef SPARTA_KOKKOS_FIXED_LISTS + + // restore() writes no member of a model today, only deep_copies into Views + // the model already holds, but the buffers are pushed for the same reason + // backup() pushes them: the device image must never trail the host one + + sr_buf_sync(k_sr_global,d_sr_global); + sr_buf_sync(k_sr_prob,d_sr_prob); + sr_buf_sync(k_sr_adsorb,d_sr_adsorb); +#endif } Kokkos::deep_copy(d_scalars,0); diff --git a/src/KOKKOS/surf_collide_td_kokkos.h b/src/KOKKOS/surf_collide_td_kokkos.h index 9601870b9..b5cf456bd 100644 --- a/src/KOKKOS/surf_collide_td_kokkos.h +++ b/src/KOKKOS/surf_collide_td_kokkos.h @@ -31,6 +31,7 @@ SurfCollideStyle(td/kk,SurfCollideTDKokkos) #include "fix_vibmode_kokkos.h" #include "surf_react_global_kokkos.h" #include "surf_react_prob_kokkos.h" +#include "surf_react_adsorb_kokkos.h" namespace SPARTA_NS { @@ -90,10 +91,23 @@ class SurfCollideTDKokkos : public SurfCollideTD { KKCopy fix_ambi_kk_copy; KKCopy fix_vibmode_kk_copy; + // the active surf react models this model may dispatch to, partitioned by + // style. Two representations, selected by SPARTA_KOKKOS_FIXED_LISTS (see + // kokkos_type.h); the device dispatch site in collide_kokkos() below is + // written once, against the KK_SR_* accessors defined there. + +#ifdef SPARTA_KOKKOS_FIXED_LISTS int sr_type_list[KOKKOS_MAX_TOT_SURF_REACT]; int sr_map[KOKKOS_MAX_TOT_SURF_REACT]; KKCopy sr_kk_global_copy[KOKKOS_MAX_SURF_REACT_PER_TYPE]; KKCopy sr_kk_prob_copy[KOKKOS_MAX_SURF_REACT_PER_TYPE]; + KKCopy sr_kk_adsorb_copy[KOKKOS_MAX_SURF_REACT_PER_TYPE]; +#else + DAT::tdual_int_1d k_sr_type_list,k_sr_map; + DAT::t_int_1d d_sr_type_list,d_sr_map; + DAT::tdual_char_1d k_sr_global,k_sr_prob,k_sr_adsorb; + DAT::t_char_1d d_sr_global,d_sr_prob,d_sr_adsorb; +#endif public: @@ -132,14 +146,17 @@ class SurfCollideTDKokkos : public SurfCollideTD { if (REACT && isr >= 0) { if (ambi_flag || vibmode_flag) memcpy(&iorig,ip,sizeof(Particle::OnePart)); - int sr_type = sr_type_list[isr]; - int m = sr_map[isr]; + int sr_type = KK_SR_TYPE(isr); + int m = KK_SR_MAP(isr); if (sr_type == 0) { - reaction = sr_kk_global_copy[m].obj. + reaction = KK_SR_GLOBAL(m). react_kokkos(ip,isurf,norm,jp,velreset,d_retry,d_nlocal); } else if (sr_type == 1) { - reaction = sr_kk_prob_copy[m].obj. + reaction = KK_SR_PROB(m). + react_kokkos(ip,isurf,norm,jp,velreset,d_retry,d_nlocal); + } else if (sr_type == 2) { + reaction = KK_SR_ADSORB(m). react_kokkos(ip,isurf,norm,jp,velreset,d_retry,d_nlocal); } diff --git a/src/KOKKOS/surf_collide_transparent_kokkos.cpp b/src/KOKKOS/surf_collide_transparent_kokkos.cpp index 855e4eaeb..e77f5c6f7 100644 --- a/src/KOKKOS/surf_collide_transparent_kokkos.cpp +++ b/src/KOKKOS/surf_collide_transparent_kokkos.cpp @@ -56,3 +56,22 @@ void SurfCollideTransparentKokkos::post_collide() auto sc = surf->sc[m]; sc->nsingle += h_nsingle(); } + +/* ---------------------------------------------------------------------- + nothing to save: this model keeps no state across a move beyond its + collision counter, which restore() rewinds +------------------------------------------------------------------------- */ + +void SurfCollideTransparentKokkos::backup() {} + +/* ---------------------------------------------------------------------- + a retried move re-runs every collision this model already counted, so the + counter has to go back to zero. Without this the retried attempt adds + on top of the aborted one and SurfCollide::nsingle -- reported as + "Surface-collisions/particle/step" -- comes out too high +------------------------------------------------------------------------- */ + +void SurfCollideTransparentKokkos::restore() +{ + Kokkos::deep_copy(d_nsingle,0); +} diff --git a/src/KOKKOS/surf_collide_transparent_kokkos.h b/src/KOKKOS/surf_collide_transparent_kokkos.h index 3a5d36676..27f7b4312 100644 --- a/src/KOKKOS/surf_collide_transparent_kokkos.h +++ b/src/KOKKOS/surf_collide_transparent_kokkos.h @@ -35,6 +35,8 @@ class SurfCollideTransparentKokkos : public SurfCollideTransparent { void pre_collide(); void post_collide(); + void backup(); + void restore(); private: diff --git a/src/KOKKOS/surf_collide_vanish_kokkos.cpp b/src/KOKKOS/surf_collide_vanish_kokkos.cpp index 9587e23f4..b503efab0 100644 --- a/src/KOKKOS/surf_collide_vanish_kokkos.cpp +++ b/src/KOKKOS/surf_collide_vanish_kokkos.cpp @@ -57,3 +57,22 @@ void SurfCollideVanishKokkos::post_collide() auto sc = surf->sc[m]; sc->nsingle += h_nsingle(); } + +/* ---------------------------------------------------------------------- + nothing to save: this model keeps no state across a move beyond its + collision counter, which restore() rewinds +------------------------------------------------------------------------- */ + +void SurfCollideVanishKokkos::backup() {} + +/* ---------------------------------------------------------------------- + a retried move re-runs every collision this model already counted, so the + counter has to go back to zero. Without this the retried attempt adds + on top of the aborted one and SurfCollide::nsingle -- reported as + "Surface-collisions/particle/step" -- comes out too high +------------------------------------------------------------------------- */ + +void SurfCollideVanishKokkos::restore() +{ + Kokkos::deep_copy(d_nsingle,0); +} diff --git a/src/KOKKOS/surf_collide_vanish_kokkos.h b/src/KOKKOS/surf_collide_vanish_kokkos.h index e04843657..ec11885c9 100644 --- a/src/KOKKOS/surf_collide_vanish_kokkos.h +++ b/src/KOKKOS/surf_collide_vanish_kokkos.h @@ -34,6 +34,8 @@ class SurfCollideVanishKokkos : public SurfCollideVanish { ~SurfCollideVanishKokkos() {} void pre_collide(); void post_collide(); + void backup(); + void restore(); private: diff --git a/src/KOKKOS/surf_kokkos.cpp b/src/KOKKOS/surf_kokkos.cpp index e1ca62fb5..b076789dc 100644 --- a/src/KOKKOS/surf_kokkos.cpp +++ b/src/KOKKOS/surf_kokkos.cpp @@ -30,6 +30,8 @@ #include "sparta_masks.h" #include "kokkos.h" +#include + using namespace SPARTA_NS; using namespace MathConst; @@ -41,7 +43,6 @@ enum{LT,LE,GT,GE,EQ,NEQ,BETWEEN}; #define DELTA 4 #define EPSSQ 1.0e-12 #define BIG 1.0e20 -#define MAXGROUP 32 /* ---------------------------------------------------------------------- */ @@ -205,8 +206,8 @@ void SurfKokkos::grow_own(int old) if (mylines == NULL) surf_kk->k_mylines = tdual_line_1d("surf:mylines",nown); else { - surf_kk->sync(Host,LINE_MASK); - surf_kk->modify(Host,LINE_MASK); // force resize on host + surf_kk->sync(Host,MYLINE_MASK); + surf_kk->modify(Host,MYLINE_MASK); // force resize on host surf_kk->k_mylines.resize(nown); } mylines = surf_kk->k_mylines.view_host().data(); @@ -214,8 +215,8 @@ void SurfKokkos::grow_own(int old) if (mytris == NULL) surf_kk->k_mytris = tdual_tri_1d("surf:mytris",nown); else { - surf_kk->sync(Host,TRI_MASK); - surf_kk->modify(Host,TRI_MASK); // force resize on host + surf_kk->sync(Host,MYTRI_MASK); + surf_kk->modify(Host,MYTRI_MASK); // force resize on host surf_kk->k_mytris.resize(nown); } mytris = surf_kk->k_mytris.view_host().data(); @@ -236,6 +237,8 @@ void SurfKokkos::sync(ExecutionSpace space, unsigned int mask) modify(Host,mask); if (mask & LINE_MASK) k_lines.sync_device(); if (mask & TRI_MASK) k_tris.sync_device(); + if (mask & MYLINE_MASK) k_mylines.sync_device(); + if (mask & MYTRI_MASK) k_mytris.sync_device(); if (mask & CUSTOM_MASK) { if (ncustom) { if (ncustom_ivec) { @@ -270,6 +273,8 @@ void SurfKokkos::sync(ExecutionSpace space, unsigned int mask) } else { if (mask & LINE_MASK) k_lines.sync_host(); if (mask & TRI_MASK) k_tris.sync_host(); + if (mask & MYLINE_MASK) k_mylines.sync_host(); + if (mask & MYTRI_MASK) k_mytris.sync_host(); if (mask & CUSTOM_MASK) { if (ncustom_ivec) { for (int i = 0; i < ncustom_ivec; i++) { @@ -313,6 +318,8 @@ void SurfKokkos::modify(ExecutionSpace space, unsigned int mask) if (space == Device) { if (mask & LINE_MASK) k_lines.modify_device(); if (mask & TRI_MASK) k_tris.modify_device(); + if (mask & MYLINE_MASK) k_mylines.modify_device(); + if (mask & MYTRI_MASK) k_mytris.modify_device(); if (mask & CUSTOM_MASK) { if (ncustom) { if (ncustom_ivec) @@ -330,6 +337,27 @@ void SurfKokkos::modify(ExecutionSpace space, unsigned int mask) if (ncustom_darray) for (int i = 0; i < ncustom_darray; i++) k_edarray.view_host()[i].k_view.modify_device(); + + // sync() reads the spread (_local) views under this same mask, so a + // claim that marked only the owned ones could never be matched: a + // host write to a spread array stayed unclaimed and the following + // sync_device copied nothing. Mark both halves. + + if (ncustom_ivec) + for (int i = 0; i < ncustom_ivec; i++) + k_eivec_local.view_host()[i].k_view.modify_device(); + + if (ncustom_iarray) + for (int i = 0; i < ncustom_iarray; i++) + k_eiarray_local.view_host()[i].k_view.modify_device(); + + if (ncustom_dvec) + for (int i = 0; i < ncustom_dvec; i++) + k_edvec_local.view_host()[i].k_view.modify_device(); + + if (ncustom_darray) + for (int i = 0; i < ncustom_darray; i++) + k_edarray_local.view_host()[i].k_view.modify_device(); } } @@ -338,6 +366,8 @@ void SurfKokkos::modify(ExecutionSpace space, unsigned int mask) } else { if (mask & LINE_MASK) k_lines.modify_host(); if (mask & TRI_MASK) k_tris.modify_host(); + if (mask & MYLINE_MASK) k_mylines.modify_host(); + if (mask & MYTRI_MASK) k_mytris.modify_host(); if (mask & CUSTOM_MASK) { if (ncustom) { if (ncustom_ivec) @@ -355,7 +385,75 @@ void SurfKokkos::modify(ExecutionSpace space, unsigned int mask) if (ncustom_darray) for (int i = 0; i < ncustom_darray; i++) k_edarray.view_host()[i].k_view.modify_host(); + + // sync() reads the spread (_local) views under this same mask, so a + // claim that marked only the owned ones could never be matched: a + // host write to a spread array stayed unclaimed and the following + // sync_device copied nothing. Mark both halves. + + if (ncustom_ivec) + for (int i = 0; i < ncustom_ivec; i++) + k_eivec_local.view_host()[i].k_view.modify_host(); + + if (ncustom_iarray) + for (int i = 0; i < ncustom_iarray; i++) + k_eiarray_local.view_host()[i].k_view.modify_host(); + + if (ncustom_dvec) + for (int i = 0; i < ncustom_dvec; i++) + k_edvec_local.view_host()[i].k_view.modify_host(); + + if (ncustom_darray) + for (int i = 0; i < ncustom_darray; i++) + k_edarray_local.view_host()[i].k_view.modify_host(); } } } } + +/* ---------------------------------------------------------------------- + memory usage of Kokkos-managed data + Surf::memory_usage() is deliberately not called: the lines/tris and + mylines/mytris counts it computes describe the host mirrors of the + DualViews below. its one non-Kokkos term, the per-surf int array in + the all-surfs branch, is carried over here +------------------------------------------------------------------------- */ + +bigint SurfKokkos::memory_usage() +{ + const bool device_distinct = + !std::is_same::value; + + bigint bytes = 0; + if (!implicit && !distributed) bytes += (bigint) nlocal * sizeof(int); + + bytes += MemKK::memory_usage(k_lines.view_host()); + bytes += MemKK::memory_usage(k_tris.view_host()); + bytes += MemKK::memory_usage(k_mylines.view_host()); + bytes += MemKK::memory_usage(k_mytris.view_host()); + for (int i = 0; i < ncustom_ivec; i++) + bytes += MemKK::memory_usage(k_eivec.view_host()[i].k_view.view_host()); + for (int i = 0; i < ncustom_iarray; i++) + bytes += MemKK::memory_usage(k_eiarray.view_host()[i].k_view.view_host()); + for (int i = 0; i < ncustom_dvec; i++) + bytes += MemKK::memory_usage(k_edvec.view_host()[i].k_view.view_host()); + for (int i = 0; i < ncustom_darray; i++) + bytes += MemKK::memory_usage(k_edarray.view_host()[i].k_view.view_host()); + + if (device_distinct) { + bytes += MemKK::memory_usage(k_lines.view_device()); + bytes += MemKK::memory_usage(k_tris.view_device()); + bytes += MemKK::memory_usage(k_mylines.view_device()); + bytes += MemKK::memory_usage(k_mytris.view_device()); + for (int i = 0; i < ncustom_ivec; i++) + bytes += MemKK::memory_usage(k_eivec.view_host()[i].k_view.view_device()); + for (int i = 0; i < ncustom_iarray; i++) + bytes += MemKK::memory_usage(k_eiarray.view_host()[i].k_view.view_device()); + for (int i = 0; i < ncustom_dvec; i++) + bytes += MemKK::memory_usage(k_edvec.view_host()[i].k_view.view_device()); + for (int i = 0; i < ncustom_darray; i++) + bytes += MemKK::memory_usage(k_edarray.view_host()[i].k_view.view_device()); + } + + return bytes; +} diff --git a/src/KOKKOS/surf_kokkos.h b/src/KOKKOS/surf_kokkos.h index a0ec10c78..574927c2a 100644 --- a/src/KOKKOS/surf_kokkos.h +++ b/src/KOKKOS/surf_kokkos.h @@ -32,6 +32,7 @@ class SurfKokkos : public Surf { void grow_own(int) override; void sync(ExecutionSpace, unsigned int); void modify(ExecutionSpace, unsigned int); + bigint memory_usage() override; int add_custom(char *, int, int) override; void allocate_custom(int) override; diff --git a/src/KOKKOS/surf_react_adsorb_kokkos.cpp b/src/KOKKOS/surf_react_adsorb_kokkos.cpp index bd96f7b3b..422bd39ef 100644 --- a/src/KOKKOS/surf_react_adsorb_kokkos.cpp +++ b/src/KOKKOS/surf_react_adsorb_kokkos.cpp @@ -19,6 +19,7 @@ #include "update.h" #include "collide.h" #include "surf.h" +#include "surf_kokkos.h" #include "surf_collide.h" #include "random_knuth.h" #include "comm.h" @@ -44,11 +45,6 @@ static void cmodel_sizes(int model, int &nc, int &nf) } } -static bool cmodel_unsupported(int m) -{ - return (m == SRA_KK::ADIABATIC || m == SRA_KK::IMPULSIVE); -} - /* ---------------------------------------------------------------------- */ SurfReactAdsorbKokkos::SurfReactAdsorbKokkos(SPARTA *sparta, int narg, char **arg) : @@ -110,20 +106,15 @@ void SurfReactAdsorbKokkos::init() { SurfReactAdsorb::init(); - // Kokkos GS adsorb currently supports a restricted feature set; - // error clearly at init rather than silently producing wrong results - - - for (int i = 0; i < nlist_gs; i++) { - OneReaction_GS *r = &rlist_gs[i]; - if (!r->active) continue; - // post-reaction collision model (cmodel) scatter on device supports - // NOMODEL/SPECULAR/DIFFUSE/CLL/TD; adiabatic/impulsive deferred + // SurfReactAdsorb::init()/init_surf() populate the per-surf custom arrays + // (area, weight, tau) on the host through surf->edvec + // (surf_react_adsorb.cpp:551-552, :460-461, :531). That is a one-time + // setup write and nothing claims it, so the first sync(Device,CUSTOM_MASK) + // copies nothing and the device starts from whatever add_custom() left -- + // reported by the watch detector as "surf:dvector ... the device keeps + // stale data" at element 0. - if (cmodel_unsupported(r->cmodel_ip) || cmodel_unsupported(r->cmodel_jp)) - error->all(FLERR,"Kokkos surf_react adsorb does not yet support reactions with " - "an adiabatic or impulsive post-reaction collision model"); - } + ((SurfKokkos *) surf)->modify(Host,CUSTOM_MASK); Kokkos::deep_copy(d_nsingle,0); Kokkos::deep_copy(d_tally_single,0); @@ -142,6 +133,24 @@ void SurfReactAdsorbKokkos::init() matches the host SurfCollide::wrapper bit-for-bit (EXACT serial) ------------------------------------------------------------------------- */ +/* ---------------------------------------------------------------------- + SurfReactAdsorb::grid_changed() re-spreads the per-surf custom values to + the local lines/tris on the host (surf_react_adsorb.cpp:1262-1266). Like + init(), nothing claims that write, so the following + sync(Device,CUSTOM_MASK) copies nothing and the device keeps the values + from before the grid changed. Reported by the watch detector as + "surf/spread:edarray_local_array ... the device keeps stale data" on the + three ps and gs_ps adsorb decks. +------------------------------------------------------------------------- */ + +void SurfReactAdsorbKokkos::grid_changed() +{ + SurfReactAdsorb::grid_changed(); + ((SurfKokkos *) surf)->modify(Host,CUSTOM_MASK); +} + +/* ---------------------------------------------------------------------- */ + void SurfReactAdsorbKokkos::init_cmodels_kokkos() { int nr = MAX(nlist_gs,1); @@ -199,8 +208,27 @@ void SurfReactAdsorbKokkos::init_reactions_gs_kokkos() h_reactions_n(i) = n; nmax = MAX(nmax,n); } + + // scratch for the per-reaction probability list react_kokkos() builds, in + // whichever of the two representations this build selected (see the + // SRA_KK_MAXPERSPECIES comment in the header) + +#ifdef SPARTA_KOKKOS_FIXED_LISTS if (nmax > SRA_KK_MAXPERSPECIES) error->all(FLERR,"Too many Kokkos surf_react adsorb reactions per species"); +#else + + // one row per concurrent thread, claimed by prob_token.acquire() for the + // duration of a react_kokkos() call. prob_token.size() is the upper bound + // acquire() can return: max_hardware_threads() on a host backend, but + // maxThreadsPerMultiProcessor*multiProcessorCount on a GPU, so this + // allocation is measured in tens of MB there, not in KB. MAX(nmax,1) + // keeps the view non-null for a PS-only run (no GS reactions, so + // react_kokkos() returns before it indexes a row) + + d_prob = Kokkos::View("sra:prob", + prob_token.size(),MAX(nmax,1)); +#endif d_list = DAT::t_int_2d("surf_react_adsorb:list",nspecies,MAX(nmax,1)); auto h_list = Kokkos::create_mirror_view(d_list); @@ -484,6 +512,23 @@ void SurfReactAdsorbKokkos::tally_update() SurfReactAdsorb::tally_update(); + // update_state_face()/update_state_surf() above rewrite the per-surf custom + // arrays (surf->edvec_local, surf_react_adsorb.cpp:612-613) on the host. + // Nothing here claimed them, so a later SurfKokkos::sync(Device, + // CUSTOM_MASK) finds clean counters, copies nothing, and the device keeps + // the previous step's surface state -- invisible where both sides share + // one allocation, wrong on a GPU. The watch detector reports it as + // "surf:dvector ... this sync_device has nothing to copy". + // Go through the SurfKokkos wrapper, not the dual views directly, so the + // mask reaches the custom arrays. + // Only on a sync step: SurfReactAdsorb::tally_update() returns at its top + // on every other one (surf_react_adsorb.cpp:1194), so nothing has touched + // the custom arrays and claiming them would make the next + // sync(Device,CUSTOM_MASK) re-upload the owned *and* spread arrays for + // nothing. + + if (sync_step) ((SurfKokkos *) surf)->modify(Host,CUSTOM_MASK); + // PS chemistry may have appended particles on the host if (psflag && sync_step) particle_kk->modify(Host,PARTICLE_MASK); diff --git a/src/KOKKOS/surf_react_adsorb_kokkos.h b/src/KOKKOS/surf_react_adsorb_kokkos.h index 7d98edf95..3785814cf 100644 --- a/src/KOKKOS/surf_react_adsorb_kokkos.h +++ b/src/KOKKOS/surf_react_adsorb_kokkos.h @@ -44,17 +44,38 @@ namespace SRA_KK { #define SRA_KK_MAXREACTANT 5 #define SRA_KK_MAXPRODUCT 5 #define SRA_KK_MAXCOEFF 4 -#define SRA_KK_MAXPERSPECIES 16 // max GS reactions a single species can be in #define SRA_KK_MAXMODELS 7 // = MAXMODELS #define SRA_KK_MAXCMCOEFF 11 // max cmodel coeffs (impulsive) #define SRA_KK_MAXCMFLAG 4 // max cmodel flags (impulsive) +// react_kokkos() needs one scratch probability per GS reaction the incident +// species takes part in. Two representations, selected by +// SPARTA_KOKKOS_FIXED_LISTS (see kokkos_type.h): +// - default: a runtime-sized device buffer, one row per concurrent thread, +// with the row claimed by a UniqueToken for the duration of the call. +// No cap on the number of GS reactions a species may appear in. +// - SPARTA_KOKKOS_FIXED_LISTS: the original per-thread stack array, capped +// at SRA_KK_MAXPERSPECIES entries and rejected at init above that. +// Unlike the tally-compute lists, this buffer sits in the inner loop of a +// per-particle kernel: the stack array is small enough to be kept in +// registers, while the device buffer is a global memory round trip plus an +// acquire/release pair on every surf collision, and its footprint is +// concurrency*nmax doubles. The buffer may well be the slower of the two on +// an accelerator; neither has been measured there. Both are kept so the two +// can be compared on real hardware by rebuilding with +// -DSPARTA_KOKKOS_FIXED_LISTS, rather than by reverting commits. + +#ifdef SPARTA_KOKKOS_FIXED_LISTS +#define SRA_KK_MAXPERSPECIES 16 // max GS reactions a single species can be in +#endif + class SurfReactAdsorbKokkos : public SurfReactAdsorb { public: SurfReactAdsorbKokkos(class SPARTA *, int, char **); SurfReactAdsorbKokkos(class SPARTA *); ~SurfReactAdsorbKokkos(); void init(); + void grid_changed() override; void tally_reset(); void tally_update(); @@ -69,6 +90,41 @@ class SurfReactAdsorbKokkos : public SurfReactAdsorb { DAT::t_int_1d d_reactions_n; // # of GS reactions for each species DAT::t_int_2d d_list; // per-species list of reaction indices + // per-thread scratch for the probability list react_kokkos() builds (see + // the SRA_KK_MAXPERSPECIES comment above). react_kokkos() is called once + // per particle from the surf collide kernels, not once per cell, so there + // is no small stable index to key the scratch on -- a per-particle buffer + // would be nlocal x nmax -- and the row is claimed with a UniqueToken + // instead, held for exactly as long as the RNG state is. + // Both members are used from a const KOKKOS_INLINE_FUNCTION: UniqueToken's + // acquire()/release()/size() are const, and a View hands out a writable + // reference through a const object, so neither needs to be mutable. + // KKCopy::copy() blits this class into the surf collide functors, so the + // token is memcpy'd rather than copy constructed. That is safe for the + // Global scope token for the same reason it is safe for the Views: the + // lock array it references is a Kokkos owned singleton that outlives every + // blitted copy (it is released in Kokkos::finalize, after SPARTA has + // deleted its styles). + // The element type is spelled double rather than DAT::t_float_2d on + // purpose: SPARTA_FLOAT is float in a mixed precision build, and the host + // react() sums these probabilities in double. + +#ifndef SPARTA_KOKKOS_FIXED_LISTS + typedef Kokkos::Experimental::UniqueToken< + DeviceType,Kokkos::Experimental::UniqueTokenScope::Global> sra_token_type; + sra_token_type prob_token; + Kokkos::View d_prob; // [prob_token.size()][nmax] + + // default layout, i.e. LayoutLeft on a GPU: consecutive threads then hold + // consecutive doubles for a given i, so the row access coalesces + +#define SRA_KK_PROB(i) d_prob(tid,i) +#define SRA_KK_PROB_RELEASE() prob_token.release(tid) +#else +#define SRA_KK_PROB(i) prob_value[i] +#define SRA_KK_PROB_RELEASE() ((void) 0) +#endif + DAT::t_int_1d d_type; // reaction type (DISSOCIATION,...) DAT::t_int_1d d_style; // SIMPLE or ARRHENIUS DAT::t_float_1d d_kreact; // precomputed rate coefficient @@ -196,7 +252,15 @@ class SurfReactAdsorbKokkos : public SurfReactAdsorb { double factor = fnum * d_weight[idx] / d_area[idx]; double ms_inv = factor / max_cover; + // claim the scratch row before the RNG state is drawn, and release it + // after the RNG state is freed, so the two nest on every exit path + +#ifdef SPARTA_KOKKOS_FIXED_LISTS double prob_value[SRA_KK_MAXPERSPECIES]; +#else + const int tid = prob_token.acquire(); +#endif + double sum_prob = 0.0; double scatter_prob = 0.0, correction = 1.0; int coeff_val = 1; @@ -214,7 +278,7 @@ class SurfReactAdsorbKokkos : public SurfReactAdsorb { case SRA_KK::DISSOCIATION: case SRA_KK::EXCHANGE: case SRA_KK::RECOMBINATION: - prob_value[i] = d_kreact(j); + SRA_KK_PROB(i) = d_kreact(j); break; case SRA_KK::AA: @@ -239,7 +303,7 @@ class SurfReactAdsorbKokkos : public SurfReactAdsorb { } else { S_theta = pow((1-surf_cover),d_coeff(j,coeff_val)); } - prob_value[i] = d_kreact(j)*S_theta; + SRA_KK_PROB(i) = d_kreact(j)*S_theta; break; case SRA_KK::ER: @@ -249,22 +313,22 @@ class SurfReactAdsorbKokkos : public SurfReactAdsorb { // empty-site count clamped at zero for the same reason the // coverage is clamped at full above - prob_value[i] = 2.0 * d_kreact(j) * + SRA_KK_PROB(i) = 2.0 * d_kreact(j) * MAX(maxstick - (bigint) d_total_state[idx],(bigint) 0) * ms_inv / fabs(dot); else - prob_value[i] = 2.0 * d_kreact(j) / fabs(dot); + SRA_KK_PROB(i) = 2.0 * d_kreact(j) / fabs(dot); break; } case SRA_KK::CI: - prob_value[i] = d_kreact(j); + SRA_KK_PROB(i) = d_kreact(j); if (d_energy_flag(j)) { double *v = ip->v; double dot = v[0]*norm[0]+v[1]*norm[1]+v[2]*norm[2]; double vmag_sq = v[0]*v[0]+v[1]*v[1]+v[2]*v[2]; double E_i = 0.5 * d_species[ip->ispecies].mass * vmag_sq; double cos_theta = fabs(dot) / sqrt(vmag_sq); - prob_value[i] *= pow(E_i,d_energy(j,0)) * pow(cos_theta,d_energy(j,1)); + SRA_KK_PROB(i) *= pow(E_i,d_energy(j,0)) * pow(cos_theta,d_energy(j,1)); } break; } @@ -272,16 +336,16 @@ class SurfReactAdsorbKokkos : public SurfReactAdsorb { for (int k = 1; k < d_nreactant(j); k++) { if (d_rstate(j,k) == 's') { if (d_rpart(j,k) == 0) - prob_value[i] *= stoich_pow_kk(d_total_state[idx],d_rstoich(j,k)) * + SRA_KK_PROB(i) *= stoich_pow_kk(d_total_state[idx],d_rstoich(j,k)) * pow(ms_inv,d_rstoich(j,k)); else - prob_value[i] *= stoich_pow_kk(d_species_state(idx,d_rad(j,k)), - d_rstoich(j,k)) * + SRA_KK_PROB(i) *= stoich_pow_kk(d_species_state(idx,d_rad(j,k)), + d_rstoich(j,k)) * pow(ms_inv,d_rstoich(j,k)); } } - sum_prob += prob_value[i]; + sum_prob += SRA_KK_PROB(i); } if (sum_prob > 1.0) correction = 1.0/sum_prob; @@ -292,12 +356,13 @@ class SurfReactAdsorbKokkos : public SurfReactAdsorb { if (react_prob > random_prob) { rand_pool.free_state(rand_gen); + SRA_KK_PROB_RELEASE(); return 0; } for (int i = 0; i < n; i++) { int j = d_list(ip->ispecies,i); - react_prob += prob_value[i] * correction; + react_prob += SRA_KK_PROB(i) * correction; if (react_prob <= random_prob) continue; // reaction j fires @@ -325,8 +390,8 @@ class SurfReactAdsorbKokkos : public SurfReactAdsorb { Kokkos::atomic_add(&a_species_delta(idx,d_pad(j,k)),d_pstoich(j,k)); // post-reaction particle handling, mirrors SurfReactAdsorb::react() - // cmodel post-reaction scatter currently supports NOMODEL and SPECULAR - // (validated at init); RNG-based cmodels (diffuse/cll/td/...) deferred + // cmodel post-reaction scatter supports every model the host + // readfile_gs() accepts; see scatter_cmodel() switch (d_type(j)) { @@ -350,16 +415,19 @@ class SurfReactAdsorbKokkos : public SurfReactAdsorb { if (reallocflag) { d_retry() = 1; rand_pool.free_state(rand_gen); + SRA_KK_PROB_RELEASE(); return 0; } jp = &d_particles[index]; rand_pool.free_state(rand_gen); + SRA_KK_PROB_RELEASE(); return (j + 1); } case SRA_KK::EXCHANGE: ip->ispecies = d_products(j,0); rand_pool.free_state(rand_gen); + SRA_KK_PROB_RELEASE(); return (j + 1); case SRA_KK::RECOMBINATION: @@ -368,6 +436,7 @@ class SurfReactAdsorbKokkos : public SurfReactAdsorb { case SRA_KK::CD: ip = NULL; rand_pool.free_state(rand_gen); + SRA_KK_PROB_RELEASE(); return (j + 1); case SRA_KK::DA: @@ -383,12 +452,20 @@ class SurfReactAdsorbKokkos : public SurfReactAdsorb { scatter_cmodel(ip,norm,d_cmodel_ip(j),j,0,rand_gen); if (d_pstoich(j,pj) == 2) { jp = create_particle(ip,d_products(j,pj),rand_gen,d_nlocal,d_retry); - if (!jp) { rand_pool.free_state(rand_gen); return 0; } + if (!jp) { + rand_pool.free_state(rand_gen); + SRA_KK_PROB_RELEASE(); + return 0; + } scatter_cmodel(jp,norm,d_cmodel_ip(j),j,0,rand_gen); } } else { jp = create_particle(ip,d_products(j,pj),rand_gen,d_nlocal,d_retry); - if (!jp) { rand_pool.free_state(rand_gen); return 0; } + if (!jp) { + rand_pool.free_state(rand_gen); + SRA_KK_PROB_RELEASE(); + return 0; + } scatter_cmodel(jp,norm,d_cmodel_jp(j),j,1,rand_gen); } } @@ -396,6 +473,7 @@ class SurfReactAdsorbKokkos : public SurfReactAdsorb { } if (d_cmodel_ip(j) != SRA_KK::NOMODEL) velreset = 1; rand_pool.free_state(rand_gen); + SRA_KK_PROB_RELEASE(); return (j + 1); } @@ -405,6 +483,7 @@ class SurfReactAdsorbKokkos : public SurfReactAdsorb { scatter_cmodel(ip,norm,d_cmodel_ip(j),j,0,rand_gen); if (d_cmodel_ip(j) != SRA_KK::NOMODEL) velreset = 1; rand_pool.free_state(rand_gen); + SRA_KK_PROB_RELEASE(); return (j + 1); case SRA_KK::CI: @@ -414,29 +493,39 @@ class SurfReactAdsorbKokkos : public SurfReactAdsorb { if (d_nprod_g_tot(j) == 2) { if (d_pstoich(j,0) == 2) { jp = create_particle(ip,d_products(j,0),rand_gen,d_nlocal,d_retry); - if (!jp) { rand_pool.free_state(rand_gen); return 0; } + if (!jp) { + rand_pool.free_state(rand_gen); + SRA_KK_PROB_RELEASE(); + return 0; + } scatter_cmodel(jp,norm,d_cmodel_ip(j),j,0,rand_gen); } else { jp = create_particle(ip,d_products(j,1),rand_gen,d_nlocal,d_retry); - if (!jp) { rand_pool.free_state(rand_gen); return 0; } + if (!jp) { + rand_pool.free_state(rand_gen); + SRA_KK_PROB_RELEASE(); + return 0; + } scatter_cmodel(jp,norm,d_cmodel_jp(j),j,1,rand_gen); } } if (d_cmodel_ip(j) != SRA_KK::NOMODEL) velreset = 1; rand_pool.free_state(rand_gen); + SRA_KK_PROB_RELEASE(); return (j + 1); } } } rand_pool.free_state(rand_gen); + SRA_KK_PROB_RELEASE(); return 0; } /* ---------------------------------------------------------------------- apply a post-reaction collision model (cmodel) scatter to particle p SPECULAR mirrors SurfCollideSpecular::wrapper() (reflect, no RNG) - NOMODEL is a no-op; RNG-based cmodels are rejected at init + NOMODEL is a no-op; every other model is replicated below ------------------------------------------------------------------------- */ KOKKOS_INLINE_FUNCTION @@ -467,7 +556,28 @@ class SurfReactAdsorbKokkos : public SurfReactAdsorb { for (int k = 0; k < SRA_KK_MAXCMFLAG; k++) fl[k] = useJp ? d_cmjp_flags(j,k) : d_cmip_flags(j,k); - if (cmodel == SRA_KK::DIFFUSE) { + if (cmodel == SRA_KK::ADIABATIC) { + adiabatic_scatter(p,norm,rg); // no coeffs + } else if (cmodel == SRA_KK::IMPULSIVE) { + const double twall = cf[0]; + const int softsphere_flag = fl[0], step_flag = fl[1]; + const int double_flag = fl[2], intenergy_flag = fl[3]; + double eng_ratio = 0.0, eff_mass = 0.0, u0_a = 0.0, u0_b = 0.0; + if (softsphere_flag) { eng_ratio = cf[1]; eff_mass = cf[2]; } + else { u0_a = cf[1]; u0_b = cf[2]; } + const double var_alpha = cf[3], theta_peak = cf[4]; + const double cos_theta_pow = cf[5], cos_phi_pow = cf[6]; + int m = 7; + double step_size = 0.0, cos_theta_pow_2 = 0.0; + double rot_frac = 0.0, vib_frac = 0.0; + if (step_flag) step_size = cf[m++]; + if (double_flag) cos_theta_pow_2 = cf[m++]; + if (intenergy_flag) { rot_frac = cf[m++]; vib_frac = cf[m++]; } + impulsive_scatter(p,norm,twall,softsphere_flag,eng_ratio,eff_mass, + u0_a,u0_b,var_alpha,theta_peak,cos_theta_pow, + cos_phi_pow,step_flag,step_size,double_flag, + cos_theta_pow_2,intenergy_flag,rot_frac,vib_frac,rg); + } else if (cmodel == SRA_KK::DIFFUSE) { diffuse_scatter(p,norm,cf[0],cf[1],rg); // tsurf, acc } else if (cmodel == SRA_KK::CLL) { cll_scatter(p,norm,cf[0],cf[1],cf[2],cf[3],cf[4],fl[0],cf[5],rg); @@ -548,6 +658,193 @@ class SurfReactAdsorbKokkos : public SurfReactAdsorb { return eng; } + + /* ---------------------------------------------------------------------- + SurfCollideAdiabatic::scatter_isotropic() -- isotropic reflection at the + incident speed; erot/evib are unchanged. no coeffs, no flags + ------------------------------------------------------------------------- */ + + KOKKOS_INLINE_FUNCTION + void adiabatic_scatter(Particle::OnePart *p, const double *norm, + rand_type &rg) const + { + double *v = p->v; + const double dot = MathExtraKokkos::dot3(v,norm); + + double tangent1[3],tangent2[3]; + tangent1[0] = v[0] - dot*norm[0]; + tangent1[1] = v[1] - dot*norm[1]; + tangent1[2] = v[2] - dot*norm[2]; + + if (MathExtraKokkos::lensq3(tangent1) == 0.0) { + tangent2[0] = rg.drand(); + tangent2[1] = rg.drand(); + tangent2[2] = rg.drand(); + MathExtraKokkos::cross3(norm,tangent2,tangent1); + } + + MathExtraKokkos::norm3(tangent1); + MathExtraKokkos::cross3(norm,tangent1,tangent2); + + const double vmag = MathExtraKokkos::len3(v); + const double theta = MathConst::MY_2PI*rg.drand(); + const double f_phi = rg.drand(); + const double sqrt_f_phi = sqrt(f_phi); + + const double vperp = vmag * sqrt(1.0 - f_phi); + const double vtan1 = vmag * sqrt_f_phi * sin(theta); + const double vtan2 = vmag * sqrt_f_phi * cos(theta); + + v[0] = vperp*norm[0] + vtan1*tangent1[0] + vtan2*tangent2[0]; + v[1] = vperp*norm[1] + vtan1*tangent1[1] + vtan2*tangent2[1]; + v[2] = vperp*norm[2] + vtan1*tangent1[2] + vtan2*tangent2[2]; + } + + /* ---------------------------------------------------------------------- + SurfCollideImpulsive::impulsive(), with the style's member state passed + in from this reaction's flattened cmodel coeffs/flags + ------------------------------------------------------------------------- */ + + KOKKOS_INLINE_FUNCTION + void impulsive_scatter(Particle::OnePart *p, const double *norm, + const double twall, + const int softsphere_flag, const double eng_ratio, + const double eff_mass, const double u0_a, + const double u0_b, const double var_alpha, + const double theta_peak, const double cos_theta_pow, + const double cos_phi_pow, + const int step_flag, const double step_size, + const int double_flag, const double cos_theta_pow_2, + const int intenergy_flag, const double rot_frac, + const double vib_frac, rand_type &rg) const + { + const double var_alpha_sq = var_alpha*var_alpha; + const int ispecies = p->ispecies; + const double mass = d_species[ispecies].mass; + + double *v = p->v; + const double dot = MathExtraKokkos::dot3(v,norm); + + double tangent1[3],tangent2[3]; + tangent1[0] = v[0] - dot*norm[0]; + tangent1[1] = v[1] - dot*norm[1]; + tangent1[2] = v[2] - dot*norm[2]; + + if (MathExtraKokkos::lensq3(tangent1) == 0.0) { + tangent2[0] = rg.drand(); + tangent2[1] = rg.drand(); + tangent2[2] = rg.drand(); + MathExtraKokkos::cross3(norm,tangent2,tangent1); + } + + MathExtraKokkos::norm3(tangent1); + MathExtraKokkos::cross3(norm,tangent1,tangent2); + + const double tan1 = MathExtraKokkos::dot3(v,tangent1); + const double tan2 = MathExtraKokkos::dot3(v,tangent2); + + const double v_i_mag_sq = MathExtraKokkos::lensq3(v); + const double E_i = 0.5 * mass * v_i_mag_sq; + const double theta_i = acos(-dot/sqrt(v_i_mag_sq)); + const double phi_i = atan2(tan2,tan1); + const double phi_peak = MathConst::MY_2PI - phi_i; + + double theta_f = 0.0, phi_f = 0.0; + double P = 0.0; + + while (rg.drand() > P) { + theta_f = MathConst::MY_PI2 * rg.drand(); + P = pow(cos( theta_f - theta_peak ),cos_theta_pow) * sin(theta_f); + if (double_flag) { + if (theta_f > theta_peak) + P = pow(cos( theta_f - theta_peak ),cos_theta_pow_2) * sin(theta_f); + } + if (step_flag) { + double func_step = 0.0; + const double tan_theta = tan(theta_f); + const double cotangent = 1.0/tan_theta; + if (cotangent > step_size) func_step = 1 - step_size*tan_theta; + P *= func_step; + } + } + + P = 0.0; + while (rg.drand() > P) { + phi_f = phi_peak + MathConst::MY_PI * (2*rg.drand() - 1); + P = pow(cos( 0.5*(phi_f - phi_peak) ),cos_phi_pow); + } + + if (phi_f > MathConst::MY_PI) phi_f -= MathConst::MY_2PI; + else if (phi_f < -MathConst::MY_PI) phi_f += MathConst::MY_2PI; + + double v_f_avg = 0.0; + if (softsphere_flag) { + const double mu = d_species[ispecies].molwt/eff_mass; + const double cos_khi = cos(MathConst::MY_PI - theta_i - theta_f); + const double sin_khi_sq = 1 - cos_khi*cos_khi; + const double dE = 2*mu/((mu+1)*(mu+1)) * + (1 + mu*sin_khi_sq + eng_ratio*(mu+1)/(2*mu) - + cos_khi*sqrt(1 - mu*mu*sin_khi_sq - eng_ratio*(mu + 1))); + const double E_f_avg = E_i * (1 - dE); + v_f_avg = var_alpha_sq * sqrt(mass/(2*E_f_avg)) * + (2*E_f_avg/(mass*var_alpha_sq) - 1); + } else { + v_f_avg = u0_a*twall + u0_b; + } + + const double v_f_max = 0.5 * (v_f_avg + sqrt(v_f_avg*v_f_avg + 6*var_alpha_sq)); + const double f_max = v_f_max*v_f_max*v_f_max * + exp(-(v_f_max - v_f_avg) * (v_f_max - v_f_avg)/(var_alpha_sq)); + + double v_f_mag = 0.0; + P = 0.0; + while (rg.drand() > P) { + v_f_mag = v_f_max + 3 * var_alpha * ( 2 * rg.drand() - 1 ); + P = v_f_mag*v_f_mag*v_f_mag/(f_max) * + exp(-(v_f_mag - v_f_avg)*(v_f_mag - v_f_avg)/(var_alpha_sq)); + } + + const double vperp = v_f_mag * cos(theta_f); + const double vtan1 = v_f_mag * sin(theta_f) * cos(phi_f); + const double vtan2 = v_f_mag * sin(theta_f) * sin(phi_f); + + v[0] = vperp*norm[0] + vtan1*tangent1[0] + vtan2*tangent2[0]; + v[1] = vperp*norm[1] + vtan1*tangent1[1] + vtan2*tangent2[1]; + v[2] = vperp*norm[2] + vtan1*tangent1[2] + vtan2*tangent2[2]; + + if (intenergy_flag) { + const double E_f = 0.5 * mass * v_f_mag * v_f_mag; + const double extra_energy = E_i - E_f; + + if (rotstyle_ == SRA_KK::NONE || d_species[ispecies].rotdof < 2) p->erot = 0.0; + else p->erot += rot_frac*extra_energy; + + const int vibdof = d_species[ispecies].vibdof; + if (vibstyle_ == SRA_KK::NONE || vibdof < 2) { + p->evib = 0.0; + } else { + const double *vibtemp = d_species[ispecies].vibtemp; + const double evib_val = p->evib + vib_frac*extra_energy; + if (vibstyle_ == SRA_KK::SMOOTH) { + p->evib = evib_val; + } else if (vibdof == 2) { + const int ivib = evib_val / (boltz_*vibtemp[0]); + p->evib = ivib * boltz_ * vibtemp[0]; + } else { + const int nvibmode = d_species[ispecies].nvibmode; + double tot_temp = 0.0, evib_sum = 0.0; + for (int imode = 0; imode < nvibmode; imode++) + tot_temp += vibtemp[imode]; + for (int imode = 0; imode < nvibmode; imode++) { + const int ivib = evib_val / (boltz_*tot_temp); + evib_sum += ivib * boltz_ * vibtemp[imode]; + } + p->evib = evib_sum; + } + } + } + } + KOKKOS_INLINE_FUNCTION void diffuse_scatter(Particle::OnePart *p, const double *norm, double twall, double acc, rand_type &rg) const diff --git a/src/KOKKOS/update_kokkos.cpp b/src/KOKKOS/update_kokkos.cpp index 1fbecfb16..8d4fef508 100644 --- a/src/KOKKOS/update_kokkos.cpp +++ b/src/KOKKOS/update_kokkos.cpp @@ -70,33 +70,74 @@ enum{BCSTD,BCWRAP,BCMIRROR,BCEXIT}; // Update::bcopt values #define VAL_1(X) X #define VAL_2(X) VAL_1(X), VAL_1(X) +/* ---------------------------------------------------------------------- + blit one active tally compute into its per-type device buffer + same operation and same rationale as KKCopy::copy() (kokkos_copy.h:71): + the object is only read on device, through KOKKOS_INLINE_FUNCTION + members, so its vtable pointer is never used and the View handles it + carries stay alive in the original the compute list holds +------------------------------------------------------------------------- */ + +#ifndef SPARTA_KOKKOS_FIXED_LISTS +namespace { + + template + void tally_buf_resize(DAT::tdual_char_1d &k, DAT::t_char_1d &d, int n) + { + const size_t need = (size_t) MAX(n,1) * sizeof(T); + if (k.view_device().extent(0) < need) { + k = DAT::tdual_char_1d("update:tally_models",need); + d = k.view_device(); + } + } + + template + void tally_buf_blit(DAT::tdual_char_1d &k, int slot, T *obj) + { + char *dst = k.view_host().data() + (size_t) slot*sizeof(T); + memcpy((void*) dst, (const void*) obj, sizeof(T)); + ((T *) dst)->copy = 1; + } + + void tally_buf_sync(DAT::tdual_char_1d &k, DAT::t_char_1d &d) + { + if (k.view_device().extent(0) == 0) return; + k.modify_host(); + k.sync_device(); + d = k.view_device(); + } +} +#endif + +/* ---------------------------------------------------------------------- */ + + /* ---------------------------------------------------------------------- */ UpdateKokkos::UpdateKokkos(SPARTA *sparta) : Update(sparta), grid_kk_copy(sparta), - domain_kk_copy(sparta), + domain_kk_copy(sparta) +#ifdef SPARTA_KOKKOS_FIXED_LISTS // Virtual functions are not yet supported on the GPU, which leads to pain: - sc_kk_specular_copy{VAL_2(KKCopy(sparta))}, - sc_kk_diffuse_copy{VAL_2(KKCopy(sparta))}, - sc_kk_vanish_copy{VAL_2(KKCopy(sparta))}, - sc_kk_piston_copy{VAL_2(KKCopy(sparta))}, - sc_kk_transparent_copy{VAL_2(KKCopy(sparta))}, - sc_kk_adiabatic_copy{VAL_2(KKCopy(sparta))}, - sc_kk_impulsive_copy{VAL_2(KKCopy(sparta))}, - sc_kk_td_copy{VAL_2(KKCopy(sparta))}, - sc_kk_cll_copy{VAL_2(KKCopy(sparta))}, - blist_active_copy{VAL_2(KKCopy(sparta))}, - slist_active_copy{VAL_2(KKCopy(sparta))}, - slist_active_isurf_copy{VAL_2(KKCopy(sparta))}, - slist_active_react_isurf_copy{VAL_2(KKCopy(sparta))}, - slist_active_react_surf_copy{VAL_2(KKCopy(sparta))}, - tmp_compute_boundary_kk(sparta), - tmp_compute_surf_kk(sparta), - tmp_compute_isurf_grid_kk(sparta), - tmp_compute_react_isurf_grid_kk(sparta), - tmp_compute_react_surf_kk(sparta) + , slist_active_copy{VAL_2(KKCopy(sparta))} + , slist_active_isurf_copy{VAL_2(KKCopy(sparta))} + , slist_active_coll_tally_copy{VAL_2(KKCopy(sparta))} + , slist_active_react_tally_copy{VAL_2(KKCopy(sparta))} + , slist_active_react_isurf_copy{VAL_2(KKCopy(sparta))} + , slist_active_react_surf_copy{VAL_2(KKCopy(sparta))} + , blist_active_copy{VAL_2(KKCopy(sparta))} + , blist_active_react_copy{VAL_2(KKCopy(sparta))} + , tmp_compute_boundary_kk(sparta) + , tmp_compute_react_boundary_kk(sparta) + , tmp_compute_surf_kk(sparta) + , tmp_compute_isurf_grid_kk(sparta) + , tmp_compute_react_isurf_grid_kk(sparta) + , tmp_compute_react_surf_kk(sparta) +#endif { nslist_surf = nslist_isurf = nslist_react_isurf = nslist_react_surf = 0; + nslist_coll_tally = nslist_react_tally = 0; + nsc_index_cached = -1; // the Kokkos views of Particle/Grid/Surf are populated from the host data // once, by setup() when prewrap is set, which then clears prewrap @@ -125,6 +166,7 @@ UpdateKokkos::UpdateKokkos(SPARTA *sparta) : Update(sparta), d_error_flag = Kokkos::subview(d_scalars,4); d_retry = Kokkos::subview(d_scalars,5); d_nlocal = Kokkos::subview(d_scalars,6); + d_tally_overflow = Kokkos::subview(d_scalars,7); d_ncomm_one = Kokkos::subview(d_scalars_big,0); d_nexit_one = Kokkos::subview(d_scalars_big,1); @@ -141,6 +183,7 @@ UpdateKokkos::UpdateKokkos(SPARTA *sparta) : Update(sparta), h_error_flag = Kokkos::subview(h_scalars,4); h_retry = Kokkos::subview(h_scalars,5); h_nlocal = Kokkos::subview(h_scalars,6); + h_tally_overflow = Kokkos::subview(h_scalars,7); h_ncomm_one = Kokkos::subview(h_scalars_big,0); h_nexit_one = Kokkos::subview(h_scalars_big,1); @@ -172,6 +215,11 @@ UpdateKokkos::~UpdateKokkos() void UpdateKokkos::init() { + // the surf_collide style list is fixed within a run but can change between + // them, so force setup_surf_collide_models() to rebuild its index maps + + nsc_index_cached = -1; + // init the UpdateKokkos class if performing a run, else just return // only set first_update if a run is being performed @@ -656,92 +704,7 @@ template < int DIM, int SURF, int REACT, int OPT > void UpdateKokkos::move() grid_kk_copy.copy(grid_kk); domain_kk_copy.copy((DomainKokkos*)domain); - if (surf->nsc > KOKKOS_MAX_TOT_SURF_COLL) - error->all(FLERR,"Kokkos currently supports a limited number of surface collide methods"); - - if (surf->nsc > 0) { - int nspec,ndiff,nvan,npist,ntrans,nadia,nimpul,ntd,ncll; - nspec = ndiff = nvan = npist = ntrans = nadia = nimpul = ntd = ncll = 0; - for (int n = 0; n < surf->nsc; n++) { - if (!surf->sc[n]->kokkosable) - error->all(FLERR,"Must use Kokkos-enabled surface collide method with Kokkos"); - if (strcmp(surf->sc[n]->style,"specular") == 0) { - if (nspec >= KOKKOS_MAX_SURF_COLL_PER_TYPE) - error->all(FLERR,"Kokkos currently supports two instances of each surface collide method"); - sc_kk_specular_copy[nspec].copy((SurfCollideSpecularKokkos*)(surf->sc[n])); - sc_kk_specular_copy[nspec].obj.pre_collide(); - sc_type_list[n] = 0; - sc_map[n] = nspec; - nspec++; - } else if (strcmp(surf->sc[n]->style,"diffuse") == 0) { - if (ndiff >= KOKKOS_MAX_SURF_COLL_PER_TYPE) - error->all(FLERR,"Kokkos currently supports two instances of each surface collide method"); - sc_kk_diffuse_copy[ndiff].copy((SurfCollideDiffuseKokkos*)(surf->sc[n])); - sc_kk_diffuse_copy[ndiff].obj.pre_collide(); - sc_type_list[n] = 1; - sc_map[n] = ndiff; - ndiff++; - } else if (strcmp(surf->sc[n]->style,"vanish") == 0) { - if (nvan >= KOKKOS_MAX_SURF_COLL_PER_TYPE) - error->all(FLERR,"Kokkos currently supports two instances of each surface collide method"); - sc_kk_vanish_copy[nvan].copy((SurfCollideVanishKokkos*)(surf->sc[n])); - sc_kk_vanish_copy[nvan].obj.pre_collide(); - sc_type_list[n] = 2; - sc_map[n] = nvan; - nvan++; - } else if (strcmp(surf->sc[n]->style,"piston") == 0) { - if (npist >= KOKKOS_MAX_SURF_COLL_PER_TYPE) - error->all(FLERR,"Kokkos currently supports two instances of each surface collide method"); - sc_kk_piston_copy[npist].copy((SurfCollidePistonKokkos*)(surf->sc[n])); - sc_kk_piston_copy[npist].obj.pre_collide(); - sc_type_list[n] = 3; - sc_map[n] = npist; - npist++; - } else if (strcmp(surf->sc[n]->style,"transparent") == 0) { - if (ntrans >= KOKKOS_MAX_SURF_COLL_PER_TYPE) - error->all(FLERR,"Kokkos currently supports two instances of each surface collide method"); - sc_kk_transparent_copy[ntrans].copy((SurfCollideTransparentKokkos*)(surf->sc[n])); - sc_kk_transparent_copy[ntrans].obj.pre_collide(); - sc_type_list[n] = 4; - sc_map[n] = ntrans; - ntrans++; - } else if (strcmp(surf->sc[n]->style,"adiabatic") == 0) { - if (nadia >= KOKKOS_MAX_SURF_COLL_PER_TYPE) - error->all(FLERR,"Kokkos currently supports two instances of each surface collide method"); - sc_kk_adiabatic_copy[nadia].copy((SurfCollideAdiabaticKokkos*)(surf->sc[n])); - sc_kk_adiabatic_copy[nadia].obj.pre_collide(); - sc_type_list[n] = 5; - sc_map[n] = nadia; - nadia++; - } else if (strcmp(surf->sc[n]->style,"impulsive") == 0) { - if (nimpul >= KOKKOS_MAX_SURF_COLL_PER_TYPE) - error->all(FLERR,"Kokkos currently supports two instances of each surface collide method"); - sc_kk_impulsive_copy[nimpul].copy((SurfCollideImpulsiveKokkos*)(surf->sc[n])); - sc_kk_impulsive_copy[nimpul].obj.pre_collide(); - sc_type_list[n] = 6; - sc_map[n] = nimpul; - nimpul++; - } else if (strcmp(surf->sc[n]->style,"td") == 0) { - if (ntd >= KOKKOS_MAX_SURF_COLL_PER_TYPE) - error->all(FLERR,"Kokkos currently supports two instances of each surface collide method"); - sc_kk_td_copy[ntd].copy((SurfCollideTDKokkos*)(surf->sc[n])); - sc_kk_td_copy[ntd].obj.pre_collide(); - sc_type_list[n] = 7; - sc_map[n] = ntd; - ntd++; - } else if (strcmp(surf->sc[n]->style,"cll") == 0) { - if (ncll >= KOKKOS_MAX_SURF_COLL_PER_TYPE) - error->all(FLERR,"Kokkos currently supports two instances of each surface collide method"); - sc_kk_cll_copy[ncll].copy((SurfCollideCLLKokkos*)(surf->sc[n])); - sc_kk_cll_copy[ncll].obj.pre_collide(); - sc_type_list[n] = 8; - sc_map[n] = ncll; - ncll++; - } else { - error->all(FLERR,"Unknown Kokkos surface collide method"); - } - } - } + setup_surf_collide_models(); Kokkos::deep_copy(h_scalars,0); Kokkos::deep_copy(h_scalars_big,0); @@ -766,10 +729,29 @@ template < int DIM, int SURF, int REACT, int OPT > void UpdateKokkos::move() h_retry() = 1; + // a per-event tally compute can force a retry of its own, and a retry + // re-runs the move over the same particles. that is only sound if the + // particle list can be rolled back first, so the backup is not gated on + // react/retry when one of those computes is active: without it the + // second attempt would move already-moved particles + + const int tally_backup = (nslist_coll_tally || nslist_react_tally); + const int do_backup = + (surf->nsr && sparta->kokkos->react_retry_flag) || tally_backup; + + // rows already tallied by earlier migration iterations of this step stay; + // an attempt of this iteration takes back only its own + + if (tally_backup) rewind_tally_computes(1); + while (h_retry()) { - if (surf->nsr && sparta->kokkos->react_retry_flag) - backup(); + if (do_backup) backup(); + + // discard the rows an aborted attempt appended, including an attempt + // repeated for a reaction overflow rather than a tally overflow + + if (tally_backup) rewind_tally_computes(0); h_retry() = 0; h_nlocal() = particle->nlocal; @@ -794,7 +776,7 @@ template < int DIM, int SURF, int REACT, int OPT > void UpdateKokkos::move() */ #if defined SPARTA_KOKKOS_GPU - #if defined(KOKKOS_ARCH_AMD_GFX940) || defined(KOKKOS_ARCH_AMD_GFX942) || defined(KOKKOS_ARCH_AMD_GFX942_APU) + #if SPARTA_KOKKOS_REDUCE_ARCH Kokkos::parallel_reduce(Kokkos::RangePolicy >(pstart,pstop),*this,reduce); #else Kokkos::parallel_for(Kokkos::RangePolicy >(pstart,pstop),*this); @@ -820,10 +802,27 @@ template < int DIM, int SURF, int REACT, int OPT > void UpdateKokkos::move() Kokkos::deep_copy(h_scalars,d_scalars); Kokkos::deep_copy(h_scalars_big,d_scalars_big); + // a per-event surf tally compute ran out of room. the row count is + // only knowable by running the move, so grow every such compute to + // what this attempt actually needed and repeat, exactly as a + // reaction overflow does. unlike a reaction overflow this needs no + // react/extra opt-in: nothing about the particle state forced it, + // and truncating a tally would silently corrupt dump tally output + + if (h_tally_overflow() && !h_retry()) { + grow_tally_computes(); + if (do_backup) restore(); + Kokkos::deep_copy(h_scalars,0); + Kokkos::deep_copy(h_scalars_big,0); + reduce = UPDATE_REDUCE(); + h_retry() = 1; + continue; + } + if (h_retry()) { int nlocal_new = h_nlocal(); - if (!sparta->kokkos->react_retry_flag) { + if (!do_backup) { error->one(FLERR,"Ran out of space for Kokkos reactions, increase react/extra" " or use react/retry"); } else @@ -884,47 +883,14 @@ template < int DIM, int SURF, int REACT, int OPT > void UpdateKokkos::move() if (error_flag) { char str[128]; - sprintf(str, + snprintf(str,sizeof(str), "Particle being sent to self proc " "on step " BIGINT_FORMAT, update->ntimestep); error->one(FLERR,str); } - if (surf->nsc > 0) { - int nspec,ndiff,nvan,npist,ntrans,nadia,nimpul,ntd,ncll; - nspec = ndiff = nvan = npist = ntrans = nadia = nimpul = ntd = ncll = 0; - for (int n = 0; n < surf->nsc; n++) { - if (strcmp(surf->sc[n]->style,"specular") == 0) { - sc_kk_specular_copy[nspec].obj.post_collide(); - nspec++; - } else if (strcmp(surf->sc[n]->style,"diffuse") == 0) { - sc_kk_diffuse_copy[ndiff].obj.post_collide(); - ndiff++; - } else if (strcmp(surf->sc[n]->style,"vanish") == 0) { - sc_kk_vanish_copy[nvan].obj.post_collide(); - nvan++; - } else if (strcmp(surf->sc[n]->style,"piston") == 0) { - sc_kk_piston_copy[npist].obj.post_collide(); - npist++; - } else if (strcmp(surf->sc[n]->style,"transparent") == 0) { - sc_kk_transparent_copy[ntrans].obj.post_collide(); - ntrans++; - } else if (strcmp(surf->sc[n]->style,"adiabatic") == 0) { - sc_kk_adiabatic_copy[nadia].obj.post_collide(); - nadia++; - } else if (strcmp(surf->sc[n]->style,"impulsive") == 0) { - sc_kk_impulsive_copy[nimpul].obj.post_collide(); - nimpul++; - } else if (strcmp(surf->sc[n]->style,"td") == 0) { - sc_kk_td_copy[ntd].obj.post_collide(); - ntd++; - } else if (strcmp(surf->sc[n]->style,"cll") == 0) { - sc_kk_cll_copy[ncll].obj.post_collide(); - ncll++; - } - } - } + for (int n = 0; n < surf->nsc; n++) sc_phase(surf->sc[n],SC_POST); // move newly created particles from surface reactions @@ -971,6 +937,11 @@ template < int DIM, int SURF, int REACT, int OPT > void UpdateKokkos::move() // END of all move/migrate iterations + // the retry-loop particle backup is reused across this step's migration + // iterations; release it now so peak memory matches the old behaviour + + free_particle_backup(); + particle->sorted = 0; particle_kk->sorted_kk = 0; @@ -1013,6 +984,12 @@ template < int DIM, int SURF, int REACT, int OPT > void UpdateKokkos::move() } else if (ComputeSurfKokkos* compute_surf_kk = dynamic_cast(slist_active[m])) { compute_surf_kk->post_surf_tally(); + } else if (ComputeSurfCollisionTallyKokkos* compute_ct_kk = + dynamic_cast(slist_active[m])) { + compute_ct_kk->post_surf_tally(); + } else if (ComputeSurfReactionTallyKokkos* compute_rt_kk = + dynamic_cast(slist_active[m])) { + compute_rt_kk->post_surf_tally(); } else { error->all(FLERR,"Kokkos does not (yet) support this surf tally compute; " "use a Kokkos-enabled surf tally compute (-sf kk)"); @@ -1020,10 +997,22 @@ template < int DIM, int SURF, int REACT, int OPT > void UpdateKokkos::move() } } + // dispatch by dynamic_cast for the same reason as the surf tally list above, + // and because compute boundary and compute react/boundary both set + // boundary_tally_flag but are unrelated classes: a static cast would call + // one's methods on the other + if (nboundary_tally) { for (int m = 0; m < nboundary_tally; m++) { - ComputeBoundaryKokkos* compute_boundary_kk = (ComputeBoundaryKokkos*)(blist_active[m]); - compute_boundary_kk->post_boundary_tally(); + if (ComputeBoundaryKokkos* c = + dynamic_cast(blist_active[m])) + c->post_boundary_tally(); + else if (ComputeReactBoundaryKokkos* c = + dynamic_cast(blist_active[m])) + c->post_boundary_tally(); + else + error->all(FLERR,"Kokkos does not (yet) support this boundary tally compute; " + "use a Kokkos-enabled boundary tally compute (-sf kk)"); } } } @@ -1257,8 +1246,11 @@ void UpdateKokkos::operator()(TagUpdateMove xnew[1] = x[1] + dtremain*v[1]; if (DIM != 2) xnew[2] = x[2] + dtremain*v[2]; if (fstyle == CFIELD) { + // DIM == 1 is the axisymmetric model, which the host treats as 2d here: + // Update::init() selects field2d on domain->dimension == 2, which is + // true for axisymmetric. Do not narrow this back to DIM == 2 if (DIM == 3) field3d(dtremain,xnew,v); - else if (DIM == 2) field2d(dtremain,xnew,v); + else field2d(dtremain,xnew,v); } else if (fstyle == PFIELD) field_per_particle(i,particle_i.icell,dtremain,xnew,v); else if (fstyle == GFIELD) field_per_grid(i,particle_i.icell,dtremain,xnew,v); } else if (pflag == PINSERT) { @@ -1267,8 +1259,11 @@ void UpdateKokkos::operator()(TagUpdateMove xnew[1] = x[1] + dtremain*v[1]; if (DIM != 2) xnew[2] = x[2] + dtremain*v[2]; if (fstyle == CFIELD) { + // DIM == 1 is the axisymmetric model, which the host treats as 2d here: + // Update::init() selects field2d on domain->dimension == 2, which is + // true for axisymmetric. Do not narrow this back to DIM == 2 if (DIM == 3) field3d(dtremain,xnew,v); - else if (DIM == 2) field2d(dtremain,xnew,v); + else field2d(dtremain,xnew,v); } else if (fstyle == PFIELD) field_per_particle(i,particle_i.icell,dtremain,xnew,v); else if (fstyle == GFIELD) field_per_grid(i,particle_i.icell,dtremain,xnew,v); } else if (pflag == PENTRY) { @@ -1842,70 +1837,16 @@ void UpdateKokkos::operator()(TagUpdateMove if (nsurf_tally) iorig = particle_i; - int n = DIM == 3 ? tri->isc : line->isc; - int sc_type = sc_type_list[n]; - int m = sc_map[n]; + const int n = DIM == 3 ? tri->isc : line->isc; if (DIM == 3) { - if (sc_type == 0) { - jpart = sc_kk_specular_copy[m].obj. - collide_kokkos(ipart,dtremain,minsurf,tri->norm,tri->isr,reaction,d_retry,d_nlocal); - } else if (sc_type == 1) { - jpart = sc_kk_diffuse_copy[m].obj. - collide_kokkos(ipart,dtremain,minsurf,tri->norm,tri->isr,reaction,d_retry,d_nlocal); - } else if (sc_type == 2) { - jpart = sc_kk_vanish_copy[m].obj. - collide_kokkos(ipart,dtremain,minsurf,tri->norm,tri->isr,reaction,d_retry,d_nlocal); - } else if (sc_type == 3) { - jpart = sc_kk_piston_copy[m].obj. - collide_kokkos(ipart,dtremain,minsurf,tri->norm,tri->isr,reaction,d_retry,d_nlocal); - } else if (sc_type == 4) { - jpart = sc_kk_transparent_copy[m].obj. - collide_kokkos(ipart,dtremain,minsurf,tri->norm,tri->isr,reaction,d_retry,d_nlocal); - } else if (sc_type == 5) { - jpart = sc_kk_adiabatic_copy[m].obj. - collide_kokkos(ipart,dtremain,minsurf,tri->norm,tri->isr,reaction,d_retry,d_nlocal); - } else if (sc_type == 6) { - jpart = sc_kk_impulsive_copy[m].obj. - collide_kokkos(ipart,dtremain,minsurf,tri->norm,tri->isr,reaction,d_retry,d_nlocal); - } else if (sc_type == 7) { - jpart = sc_kk_td_copy[m].obj. - collide_kokkos(ipart,dtremain,minsurf,tri->norm,tri->isr,reaction,d_retry,d_nlocal); - } else if (sc_type == 8) { - jpart = sc_kk_cll_copy[m].obj. - collide_kokkos(ipart,dtremain,minsurf,tri->norm,tri->isr,reaction,d_retry,d_nlocal); - } + jpart = surf_collide_dispatch + (n,ipart,dtremain,minsurf,tri->norm,tri->isr,reaction,d_retry,d_nlocal); } if (DIM != 3) { - if (sc_type == 0) { - jpart = sc_kk_specular_copy[m].obj. - collide_kokkos(ipart,dtremain,minsurf,line->norm,line->isr,reaction,d_retry,d_nlocal); - } else if (sc_type == 1) { - jpart = sc_kk_diffuse_copy[m].obj. - collide_kokkos(ipart,dtremain,minsurf,line->norm,line->isr,reaction,d_retry,d_nlocal); - } else if (sc_type == 2) { - jpart = sc_kk_vanish_copy[m].obj. - collide_kokkos(ipart,dtremain,minsurf,line->norm,line->isr,reaction,d_retry,d_nlocal); - } else if (sc_type == 3) { - jpart = sc_kk_piston_copy[m].obj. - collide_kokkos(ipart,dtremain,minsurf,line->norm,line->isr,reaction,d_retry,d_nlocal); - } else if (sc_type == 4) { - jpart = sc_kk_transparent_copy[m].obj. - collide_kokkos(ipart,dtremain,minsurf,line->norm,line->isr,reaction,d_retry,d_nlocal); - } else if (sc_type == 5) { - jpart = sc_kk_adiabatic_copy[m].obj. - collide_kokkos(ipart,dtremain,minsurf,line->norm,line->isr,reaction,d_retry,d_nlocal); - } else if (sc_type == 6) { - jpart = sc_kk_impulsive_copy[m].obj. - collide_kokkos(ipart,dtremain,minsurf,line->norm,line->isr,reaction,d_retry,d_nlocal); - } else if (sc_type == 7) { - jpart = sc_kk_td_copy[m].obj. - collide_kokkos(ipart,dtremain,minsurf,line->norm,line->isr,reaction,d_retry,d_nlocal); - } else if (sc_type == 8) { - jpart = sc_kk_cll_copy[m].obj. - collide_kokkos(ipart,dtremain,minsurf,line->norm,line->isr,reaction,d_retry,d_nlocal); - } + jpart = surf_collide_dispatch + (n,ipart,dtremain,minsurf,line->norm,line->isr,reaction,d_retry,d_nlocal); } if (jpart) { @@ -1917,17 +1858,23 @@ void UpdateKokkos::operator()(TagUpdateMove } if (nsurf_tally) { - for (m = 0; m < nslist_surf; m++) - slist_active_copy[m].obj. + for (int m = 0; m < nslist_surf; m++) + UK_SLIST_SURF(m). surf_tally_kk(dtremain,minsurf,icell,reaction,&iorig,ipart,jpart); - for (m = 0; m < nslist_isurf; m++) - slist_active_isurf_copy[m].obj. + for (int m = 0; m < nslist_isurf; m++) + UK_SLIST_ISURF(m). surf_tally_kk(dtremain,minsurf,icell,reaction,&iorig,ipart,jpart); - for (m = 0; m < nslist_react_isurf; m++) - slist_active_react_isurf_copy[m].obj. + for (int m = 0; m < nslist_coll_tally; m++) + UK_SLIST_COLL_TALLY(m). + surf_tally_kk(dtremain,minsurf,icell,reaction,&iorig,ipart,jpart); + for (int m = 0; m < nslist_react_tally; m++) + UK_SLIST_REACT_TALLY(m). + surf_tally_kk(dtremain,minsurf,icell,reaction,&iorig,ipart,jpart); + for (int m = 0; m < nslist_react_isurf; m++) + UK_SLIST_REACT_ISURF(m). surf_tally_kk(dtremain,minsurf,icell,reaction,&iorig,ipart,jpart); - for (m = 0; m < nslist_react_surf; m++) - slist_active_react_surf_copy[m].obj. + for (int m = 0; m < nslist_react_surf; m++) + UK_SLIST_REACT_SURF(m). surf_tally_kk(dtremain,minsurf,icell,reaction,&iorig,ipart,jpart); } @@ -2143,37 +2090,11 @@ void UpdateKokkos::operator()(TagUpdateMove // reset all components of xnew, in case dtremain changed // if axisymmetric, caller will reset again, including xnew[2] - int n = domain_kk_copy.obj.surf_collide[outface]; - int sc_type = sc_type_list[n]; - int m = sc_map[n]; - - if (sc_type == 0) - jpart = sc_kk_specular_copy[m].obj. - collide_kokkos(ipart,dtremain,-(outface+1),domain_kk_copy.obj.norm[outface],domain_kk_copy.obj.surf_react[outface],reaction,d_retry,d_nlocal); - else if (sc_type == 1) - jpart = sc_kk_diffuse_copy[m].obj. - collide_kokkos(ipart,dtremain,-(outface+1),domain_kk_copy.obj.norm[outface],domain_kk_copy.obj.surf_react[outface],reaction,d_retry,d_nlocal); - else if (sc_type == 2) - jpart = sc_kk_vanish_copy[m].obj. - collide_kokkos(ipart,dtremain,-(outface+1),domain_kk_copy.obj.norm[outface],domain_kk_copy.obj.surf_react[outface],reaction,d_retry,d_nlocal); - else if (sc_type == 3) - jpart = sc_kk_piston_copy[m].obj. - collide_kokkos(ipart,dtremain,-(outface+1),domain_kk_copy.obj.norm[outface],domain_kk_copy.obj.surf_react[outface],reaction,d_retry,d_nlocal); - else if (sc_type == 4) - jpart = sc_kk_transparent_copy[m].obj. - collide_kokkos(ipart,dtremain,-(outface+1),domain_kk_copy.obj.norm[outface],domain_kk_copy.obj.surf_react[outface],reaction,d_retry,d_nlocal); - else if (sc_type == 5) - jpart = sc_kk_adiabatic_copy[m].obj. - collide_kokkos(ipart,dtremain,-(outface+1),domain_kk_copy.obj.norm[outface],domain_kk_copy.obj.surf_react[outface],reaction,d_retry,d_nlocal); - else if (sc_type == 6) - jpart = sc_kk_impulsive_copy[m].obj. - collide_kokkos(ipart,dtremain,-(outface+1),domain_kk_copy.obj.norm[outface],domain_kk_copy.obj.surf_react[outface],reaction,d_retry,d_nlocal); - else if (sc_type == 7) - jpart = sc_kk_td_copy[m].obj. - collide_kokkos(ipart,dtremain,-(outface+1),domain_kk_copy.obj.norm[outface],domain_kk_copy.obj.surf_react[outface],reaction,d_retry,d_nlocal); - else if (sc_type == 8) - jpart = sc_kk_cll_copy[m].obj. - collide_kokkos(ipart,dtremain,-(outface+1),domain_kk_copy.obj.norm[outface],domain_kk_copy.obj.surf_react[outface],reaction,d_retry,d_nlocal); + const int n = domain_kk_copy.obj.surf_collide[outface]; + + jpart = surf_collide_dispatch + (n,ipart,dtremain,-(outface+1),domain_kk_copy.obj.norm[outface], + domain_kk_copy.obj.surf_react[outface],reaction,d_retry,d_nlocal); if (ipart) { double *x = ipart->x; @@ -2192,10 +2113,14 @@ void UpdateKokkos::operator()(TagUpdateMove v = particle_i.v; } - if (nboundary_tally) - for (int m = 0; m < nboundary_tally; m++) - blist_active_copy[m].obj. + if (nboundary_tally) { + for (int m = 0; m < nblist_boundary; m++) + UK_BLIST(m). + boundary_tally_kk(dtremain,outface,bflag,reaction,&iorig,ipart,jpart,domain_kk_copy.obj.norm[outface]); + for (int m = 0; m < nblist_react; m++) + UK_BLIST_REACT(m). boundary_tally_kk(dtremain,outface,bflag,reaction,&iorig,ipart,jpart,domain_kk_copy.obj.norm[outface]); + } if (DIM == 1) { xnew[0] = x[0] + dtremain*v[0]; @@ -2474,22 +2399,66 @@ void UpdateKokkos::tally_set(bigint ntimestep) int i; - if (nboundary_tally > KOKKOS_MAX_BLIST) - error->all(FLERR,"Kokkos currently only supports two instances of compute boundary"); + // dispatch by dynamic_cast, as setup_surf_tally_copies() does: compute + // boundary and compute react/boundary both set boundary_tally_flag but + // are unrelated class hierarchies, so a static cast would call one's + // methods on the other. The cast also fails for a plain compute boundary + // under "-k on" without "-sf kk", which is likewise not the Kokkos class + + // count first: the buffers have to be sized before anything is blitted in + nblist_boundary = nblist_react = 0; for (i = 0; i < nboundary_tally; i++) { - ComputeBoundaryKokkos* compute_boundary_kk = (ComputeBoundaryKokkos*)(blist_active[i]); - compute_boundary_kk->pre_boundary_tally(); - blist_active_copy[i].copy(compute_boundary_kk); + if (dynamic_cast(blist_active[i])) nblist_boundary++; + else if (dynamic_cast(blist_active[i])) nblist_react++; + else + error->all(FLERR,"Kokkos does not (yet) support this boundary tally compute; " + "use a Kokkos-enabled boundary tally compute (-sf kk)"); } - // every Kokkos functor captures the whole array by value, so the unused - // slots must not alias a compute that may be reallocated or deleted while - // they still reference count it: point them at a temporary that lives as - // long as this class +#ifdef SPARTA_KOKKOS_FIXED_LISTS + if (nblist_boundary > KOKKOS_MAX_BLIST || nblist_react > KOKKOS_MAX_BLIST) + error->all(FLERR,"Kokkos currently only supports two instances of compute boundary"); +#else + tally_buf_resize(k_blist,d_blist,nblist_boundary); + tally_buf_resize(k_blist_react,d_blist_react,nblist_react); +#endif - for (i = nboundary_tally; i < KOKKOS_MAX_BLIST; i++) + nblist_boundary = nblist_react = 0; + + for (i = 0; i < nboundary_tally; i++) { + if (ComputeBoundaryKokkos* c = + dynamic_cast(blist_active[i])) { + c->pre_boundary_tally(); +#ifdef SPARTA_KOKKOS_FIXED_LISTS + blist_active_copy[nblist_boundary].copy(c); +#else + tally_buf_blit(k_blist,nblist_boundary,c); +#endif + nblist_boundary++; + } else if (ComputeReactBoundaryKokkos* c = + dynamic_cast(blist_active[i])) { + c->pre_boundary_tally(); +#ifdef SPARTA_KOKKOS_FIXED_LISTS + blist_active_react_copy[nblist_react].copy(c); +#else + tally_buf_blit(k_blist_react,nblist_react,c); +#endif + nblist_react++; + } else + error->all(FLERR,"Kokkos does not (yet) support this boundary tally compute; " + "use a Kokkos-enabled boundary tally compute (-sf kk)"); + } + +#ifdef SPARTA_KOKKOS_FIXED_LISTS + for (i = nblist_boundary; i < KOKKOS_MAX_BLIST; i++) blist_active_copy[i].copy(&tmp_compute_boundary_kk); + for (i = nblist_react; i < KOKKOS_MAX_BLIST; i++) + blist_active_react_copy[i].copy(&tmp_compute_react_boundary_kk); +#else + tally_buf_sync(k_blist,d_blist); + tally_buf_sync(k_blist_react,d_blist_react); +#endif // surf-tally compute scatter views (slist_active_copy et al.) are // (re)established in setup_surf_tally_copies(), which run() calls after @@ -2507,68 +2476,118 @@ void UpdateKokkos::tally_set(bigint ntimestep) void UpdateKokkos::setup_surf_tally_copies() { - int i; - - // partition surf tally computes into "compute surf" (slist_active_copy) and - // "compute isurf/grid" (slist_active_isurf_copy); both tally on-device via - // surf_tally_kk(), invoked from the move kernel's surface collision loop - - nslist_surf = nslist_isurf = nslist_react_isurf = nslist_react_surf = 0; - + // partition the active surf tally computes by type, one runtime-sized + // device buffer each; all of them tally on-device via surf_tally_kk(), + // invoked from the move kernel's surface collision loop // dispatch by dynamic_cast, not by style string: the styles are also // registered under explicit "/kk" names (e.g. isurf/grid/kk), so a // style-string compare would reject a compute the user typed with the - // suffix. The four Kokkos tally computes are unrelated class hierarchies, + // suffix. The Kokkos tally computes are unrelated class hierarchies, // so the casts are mutually exclusive and order-independent. - if (nsurf_tally) { - for (i = 0; i < nsurf_tally; i++) { - if (ComputeISurfGridKokkos* compute_isurf_kk = - dynamic_cast(slist_active[i])) { - if (nslist_isurf >= KOKKOS_MAX_SLIST) - error->all(FLERR,"Kokkos currently only supports two instances of compute isurf/grid"); - compute_isurf_kk->pre_surf_tally(); - slist_active_isurf_copy[nslist_isurf].copy(compute_isurf_kk); - nslist_isurf++; - } else if (ComputeReactISurfGridKokkos* compute_react_isurf_kk = - dynamic_cast(slist_active[i])) { - if (nslist_react_isurf >= KOKKOS_MAX_SLIST) - error->all(FLERR,"Kokkos currently only supports two instances of compute react/isurf/grid"); - compute_react_isurf_kk->pre_surf_tally(); - slist_active_react_isurf_copy[nslist_react_isurf].copy(compute_react_isurf_kk); - nslist_react_isurf++; - } else if (ComputeReactSurfKokkos* compute_react_surf_kk = - dynamic_cast(slist_active[i])) { - if (nslist_react_surf >= KOKKOS_MAX_SLIST) - error->all(FLERR,"Kokkos currently only supports two instances of compute react/surf"); - compute_react_surf_kk->pre_surf_tally(); - slist_active_react_surf_copy[nslist_react_surf].copy(compute_react_surf_kk); - nslist_react_surf++; - } else if (ComputeSurfKokkos* compute_surf_kk = - dynamic_cast(slist_active[i])) { - if (nslist_surf >= KOKKOS_MAX_SLIST) - error->all(FLERR,"Kokkos currently only supports two instances of compute surface"); - compute_surf_kk->pre_surf_tally(); - slist_active_copy[nslist_surf].copy(compute_surf_kk); - nslist_surf++; - } else { - error->all(FLERR,"Kokkos does not (yet) support this surf tally compute; " - "use a Kokkos-enabled surf tally compute (-sf kk)"); - } - } + nslist_surf = nslist_isurf = nslist_react_isurf = nslist_react_surf = 0; + nslist_coll_tally = nslist_react_tally = 0; + + // count first: the buffers have to be sized before anything is blitted in + + for (int i = 0; i < nsurf_tally; i++) { + if (dynamic_cast(slist_active[i])) nslist_isurf++; + else if (dynamic_cast(slist_active[i])) nslist_react_isurf++; + else if (dynamic_cast(slist_active[i])) nslist_react_surf++; + else if (dynamic_cast(slist_active[i])) nslist_surf++; + else if (dynamic_cast(slist_active[i])) nslist_coll_tally++; + else if (dynamic_cast(slist_active[i])) nslist_react_tally++; + else + error->all(FLERR,"Kokkos does not (yet) support this surf tally compute; " + "use a Kokkos-enabled surf tally compute (-sf kk)"); } - // fill unused slots of each typed copy list with the temporary - // to avoid the copy getting stale leading to an issue with view ref counting +#ifdef SPARTA_KOKKOS_FIXED_LISTS + if (nslist_isurf > KOKKOS_MAX_SLIST || nslist_react_isurf > KOKKOS_MAX_SLIST || + nslist_react_surf > KOKKOS_MAX_SLIST || nslist_surf > KOKKOS_MAX_SLIST || + nslist_coll_tally > KOKKOS_MAX_SLIST || nslist_react_tally > KOKKOS_MAX_SLIST) + error->all(FLERR,"Kokkos currently only supports two instances of each surf tally compute"); +#else + tally_buf_resize(k_slist_isurf,d_slist_isurf,nslist_isurf); + tally_buf_resize(k_slist_react_isurf,d_slist_react_isurf,nslist_react_isurf); + tally_buf_resize(k_slist_react_surf,d_slist_react_surf,nslist_react_surf); + tally_buf_resize(k_slist_surf,d_slist_surf,nslist_surf); + tally_buf_resize(k_slist_coll_tally,d_slist_coll_tally,nslist_coll_tally); + tally_buf_resize(k_slist_react_tally,d_slist_react_tally,nslist_react_tally); +#endif + + // then run each compute's pre_surf_tally() in list order, as before, and + // blit it into its type's buffer + + int nisurf = 0, nrisurf = 0, nrsurf = 0, nsurf = 0, nct = 0, nrt = 0; + + for (int i = 0; i < nsurf_tally; i++) { + if (ComputeISurfGridKokkos* c = + dynamic_cast(slist_active[i])) { + c->pre_surf_tally(); +#ifdef SPARTA_KOKKOS_FIXED_LISTS + slist_active_isurf_copy[nisurf++].copy(c); +#else + tally_buf_blit(k_slist_isurf,nisurf++,c); +#endif + } else if (ComputeReactISurfGridKokkos* c = + dynamic_cast(slist_active[i])) { + c->pre_surf_tally(); +#ifdef SPARTA_KOKKOS_FIXED_LISTS + slist_active_react_isurf_copy[nrisurf++].copy(c); +#else + tally_buf_blit(k_slist_react_isurf,nrisurf++,c); +#endif + } else if (ComputeReactSurfKokkos* c = + dynamic_cast(slist_active[i])) { + c->pre_surf_tally(); +#ifdef SPARTA_KOKKOS_FIXED_LISTS + slist_active_react_surf_copy[nrsurf++].copy(c); +#else + tally_buf_blit(k_slist_react_surf,nrsurf++,c); +#endif + } else if (ComputeSurfKokkos* c = + dynamic_cast(slist_active[i])) { + c->pre_surf_tally(); +#ifdef SPARTA_KOKKOS_FIXED_LISTS + slist_active_copy[nsurf++].copy(c); +#else + tally_buf_blit(k_slist_surf,nsurf++,c); +#endif + } else if (ComputeSurfCollisionTallyKokkos* c = + dynamic_cast(slist_active[i])) { + c->pre_surf_tally(); + c->d_overflow = d_tally_overflow; +#ifdef SPARTA_KOKKOS_FIXED_LISTS + slist_active_coll_tally_copy[nct++].copy(c); +#else + tally_buf_blit(k_slist_coll_tally,nct++,c); +#endif + } else if (ComputeSurfReactionTallyKokkos* c = + dynamic_cast(slist_active[i])) { + c->pre_surf_tally(); + c->d_overflow = d_tally_overflow; +#ifdef SPARTA_KOKKOS_FIXED_LISTS + slist_active_react_tally_copy[nrt++].copy(c); +#else + tally_buf_blit(k_slist_react_tally,nrt++,c); +#endif + } + } - for (i = nslist_surf; i < KOKKOS_MAX_SLIST; i++) - slist_active_copy[i].copy(&tmp_compute_surf_kk); - for (i = nslist_isurf; i < KOKKOS_MAX_SLIST; i++) - slist_active_isurf_copy[i].copy(&tmp_compute_isurf_grid_kk); - for (i = nslist_react_isurf; i < KOKKOS_MAX_SLIST; i++) - slist_active_react_isurf_copy[i].copy(&tmp_compute_react_isurf_grid_kk); - for (i = nslist_react_surf; i < KOKKOS_MAX_SLIST; i++) - slist_active_react_surf_copy[i].copy(&tmp_compute_react_surf_kk); +#ifdef SPARTA_KOKKOS_FIXED_LISTS + for (int i = nsurf; i < KOKKOS_MAX_SLIST; i++) slist_active_copy[i].copy(&tmp_compute_surf_kk); + for (int i = nisurf; i < KOKKOS_MAX_SLIST; i++) slist_active_isurf_copy[i].copy(&tmp_compute_isurf_grid_kk); + for (int i = nrisurf; i < KOKKOS_MAX_SLIST; i++) slist_active_react_isurf_copy[i].copy(&tmp_compute_react_isurf_grid_kk); + for (int i = nrsurf; i < KOKKOS_MAX_SLIST; i++) slist_active_react_surf_copy[i].copy(&tmp_compute_react_surf_kk); +#else + tally_buf_sync(k_slist_isurf,d_slist_isurf); + tally_buf_sync(k_slist_react_isurf,d_slist_react_isurf); + tally_buf_sync(k_slist_react_surf,d_slist_react_surf); + tally_buf_sync(k_slist_surf,d_slist_surf); + tally_buf_sync(k_slist_coll_tally,d_slist_coll_tally); + tally_buf_sync(k_slist_react_tally,d_slist_react_tally); +#endif // gas/gas tally computes are validated and set up by CollideVSSKokkos, // which invokes their on-device gas_tally_kk() from the collision kernel @@ -2580,38 +2599,22 @@ void UpdateKokkos::backup() { ParticleKokkos* particle_kk = (ParticleKokkos*) particle; d_particles = particle_kk->k_particles.view_device(); - d_particles_backup = decltype(d_particles)(Kokkos::view_alloc("update:particles_backup",Kokkos::WithoutInitializing),d_particles.extent(0)); + + // reuse the buffer across the migration iterations of a step. backup() is + // called once per iteration of the retry loop, and the loop runs once per + // migration iteration, so reallocating here churned a full particle-sized + // allocation several times per timestep whenever a per-event surf tally + // compute was active. restore() no longer frees it; free_particle_backup() + // does, once the step's migration is done, so peak memory is unchanged. + // The extents must stay equal because restore() deep_copies between them. + + if (d_particles_backup.extent(0) != d_particles.extent(0)) + d_particles_backup = decltype(d_particles)(Kokkos::view_alloc("update:particles_backup",Kokkos::WithoutInitializing),d_particles.extent(0)); Kokkos::deep_copy(d_particles_backup,d_particles); - if (surf->nsc > 0) { - int nspec,ndiff,npist,nadia,nimpul,ntd,ncll; - nspec = ndiff = npist = nadia = nimpul = ntd = ncll = 0; - for (int n = 0; n < surf->nsc; n++) { - if (strcmp(surf->sc[n]->style,"specular") == 0) { - sc_kk_specular_copy[nspec].obj.backup(); - nspec++; - } else if (strcmp(surf->sc[n]->style,"diffuse") == 0) { - sc_kk_diffuse_copy[ndiff].obj.backup(); - ndiff++; - } else if (strcmp(surf->sc[n]->style,"piston") == 0) { - sc_kk_piston_copy[npist].obj.backup(); - npist++; - } else if (strcmp(surf->sc[n]->style,"adiabatic") == 0) { - sc_kk_adiabatic_copy[nadia].obj.backup(); - nadia++; - } else if (strcmp(surf->sc[n]->style,"impulsive") == 0) { - sc_kk_impulsive_copy[nimpul].obj.backup(); - nimpul++; - } else if (strcmp(surf->sc[n]->style,"td") == 0) { - sc_kk_td_copy[ntd].obj.backup(); - ntd++; - } else if (strcmp(surf->sc[n]->style,"cll") == 0) { - sc_kk_cll_copy[ncll].obj.backup(); - ncll++; - } - } - } + for (int n = 0; n < surf->nsc; n++) sc_phase(surf->sc[n],SC_BACKUP); + upload_surf_collide_models(); } /* ---------------------------------------------------------------------- */ @@ -2622,36 +2625,261 @@ void UpdateKokkos::restore() Kokkos::deep_copy(particle_kk->k_particles.view_device(),d_particles_backup); d_particles = particle_kk->k_particles.view_device(); - if (surf->nsc > 0) { - int nspec,ndiff,npist,nadia,nimpul,ntd,ncll; - nspec = ndiff = npist = nadia = nimpul = ntd = ncll = 0; + for (int n = 0; n < surf->nsc; n++) sc_phase(surf->sc[n],SC_RESTORE); + upload_surf_collide_models(); + + // the buffer stays allocated for the next attempt of this step; + // free_particle_backup() releases it once the step is done +} + +/* ---------------------------------------------------------------------- + release the particle backup buffer at the end of a step's migration + keeps peak memory the same as when restore() freed it, without + reallocating on every migration iteration +------------------------------------------------------------------------- */ + +void UpdateKokkos::free_particle_backup() +{ + d_particles_backup = {}; +} + +/* ---------------------------------------------------------------------- + grow every per-event surf tally compute past what the failed attempt + needed, then let the caller repeat the move +------------------------------------------------------------------------- */ + +/* ---------------------------------------------------------------------- + surf_collide model plumbing + the nine styles share no Kokkos base class -- pre_collide(), post_collide(), + backup() and restore() are declared on the concrete classes, not on + SurfCollide -- so every host-side pass over the models has to name all + nine types. It used to be spelled out four times as a strcmp ladder; + the list lives here once instead +------------------------------------------------------------------------- */ + +#define SC_FOREACH(F) \ + F(SC_SPECULAR,SurfCollideSpecularKokkos) \ + F(SC_DIFFUSE,SurfCollideDiffuseKokkos) \ + F(SC_VANISH,SurfCollideVanishKokkos) \ + F(SC_PISTON,SurfCollidePistonKokkos) \ + F(SC_TRANSPARENT,SurfCollideTransparentKokkos) \ + F(SC_ADIABATIC,SurfCollideAdiabaticKokkos) \ + F(SC_IMPULSIVE,SurfCollideImpulsiveKokkos) \ + F(SC_TD,SurfCollideTDKokkos) \ + F(SC_CLL,SurfCollideCLLKokkos) + +namespace { + + template void sc_run(SurfCollide *base, int phase) + { + T *m = (T *) base; + if (phase == SC_PRE) m->pre_collide(); + else if (phase == SC_POST) m->post_collide(); + else if (phase == SC_BACKUP) m->backup(); + else m->restore(); + } + + template void sc_blit(char *dst, SurfCollide *base) + { + memcpy((void*) dst, (const void*) ((T *) base), sizeof(T)); + + // the image in the buffer is read on device and never destructed, so + // mark it non-owning exactly as KKCopy::copy() does + + ((T *) dst)->copy = 1; + } +} + +/* ---------------------------------------------------------------------- */ + +int UpdateKokkos::surf_collide_style_tag(SurfCollide *sc) +{ + if (strcmp(sc->style,"specular") == 0) return SC_SPECULAR; + if (strcmp(sc->style,"diffuse") == 0) return SC_DIFFUSE; + if (strcmp(sc->style,"vanish") == 0) return SC_VANISH; + if (strcmp(sc->style,"piston") == 0) return SC_PISTON; + if (strcmp(sc->style,"transparent") == 0) return SC_TRANSPARENT; + if (strcmp(sc->style,"adiabatic") == 0) return SC_ADIABATIC; + if (strcmp(sc->style,"impulsive") == 0) return SC_IMPULSIVE; + if (strcmp(sc->style,"td") == 0) return SC_TD; + if (strcmp(sc->style,"cll") == 0) return SC_CLL; + return -1; +} + +/* ---------------------------------------------------------------------- */ + +void UpdateKokkos::sc_phase(SurfCollide *sc, int phase) +{ + switch (surf_collide_style_tag(sc)) { +#define SC_RUN(TAG,TYPE) case TAG: sc_run(sc,phase); break; + SC_FOREACH(SC_RUN) +#undef SC_RUN + } +} + +/* ---------------------------------------------------------------------- + size of one blitted model of each style +------------------------------------------------------------------------- */ + +size_t UpdateKokkos::sc_sizeof(int tag) +{ + switch (tag) { +#define SC_SIZE(TAG,TYPE) case TAG: return sizeof(TYPE); + SC_FOREACH(SC_SIZE) +#undef SC_SIZE + } + return 0; +} + +/* ---------------------------------------------------------------------- + count the models, run their pre_collide(), and blit them to the device + called once per move(), before the retry loop +------------------------------------------------------------------------- */ + +void UpdateKokkos::setup_surf_collide_models() +{ + if (surf->nsc == 0) { + for (int t = 0; t < SC_NSTYLE; t++) nsc_style[t] = 0; + nsc_index_cached = -1; + return; + } + + // index maps: which style each surf_collide is, and its slot within it. + // this function is called from inside move()'s migration loop, but the + // maps depend only on the surf_collide style list, which cannot change + // during a run -- so build them once and keep nsc_style[] and the host + // mirrors, rather than re-deriving and re-uploading both every iteration. + // init() clears nsc_index_cached so a new run rebuilds + + if (nsc_index_cached != surf->nsc) { + for (int t = 0; t < SC_NSTYLE; t++) nsc_style[t] = 0; + + if ((int) d_sc_type.extent(0) < surf->nsc) { + d_sc_type = DAT::t_int_1d("update:sc_type",surf->nsc); + d_sc_map = DAT::t_int_1d("update:sc_map",surf->nsc); + h_sc_type = Kokkos::create_mirror_view(d_sc_type); + h_sc_map = Kokkos::create_mirror_view(d_sc_map); + } + for (int n = 0; n < surf->nsc; n++) { - if (strcmp(surf->sc[n]->style,"specular") == 0) { - sc_kk_specular_copy[nspec].obj.restore(); - nspec++; - } else if (strcmp(surf->sc[n]->style,"diffuse") == 0) { - sc_kk_diffuse_copy[ndiff].obj.restore(); - ndiff++; - } else if (strcmp(surf->sc[n]->style,"piston") == 0) { - sc_kk_piston_copy[npist].obj.restore(); - npist++; - } else if (strcmp(surf->sc[n]->style,"adiabatic") == 0) { - sc_kk_adiabatic_copy[nadia].obj.restore(); - nadia++; - } else if (strcmp(surf->sc[n]->style,"impulsive") == 0) { - sc_kk_impulsive_copy[nimpul].obj.restore(); - nimpul++; - } else if (strcmp(surf->sc[n]->style,"td") == 0) { - sc_kk_td_copy[ntd].obj.restore(); - ntd++; - } else if (strcmp(surf->sc[n]->style,"cll") == 0) { - sc_kk_cll_copy[ncll].obj.restore(); - ncll++; - } + if (!surf->sc[n]->kokkosable) + error->all(FLERR,"Must use Kokkos-enabled surface collide method with Kokkos"); + const int tag = surf_collide_style_tag(surf->sc[n]); + if (tag < 0) error->all(FLERR,"Unknown Kokkos surface collide method"); + h_sc_type(n) = tag; + h_sc_map(n) = nsc_style[tag]++; } + + Kokkos::deep_copy(d_sc_type,h_sc_type); + Kokkos::deep_copy(d_sc_map,h_sc_map); + + nsc_index_cached = surf->nsc; } - // deallocate references to reduce memory use + // one buffer per style, grown to hold every instance of it - d_particles_backup = {}; + for (int t = 0; t < SC_NSTYLE; t++) { + if (!nsc_style[t]) continue; + const size_t need = (size_t) nsc_style[t] * sc_sizeof(t); + if (k_sc[t].view_device().extent(0) < need) { + k_sc[t] = DAT::tdual_char_1d("update:sc_models",need); + d_sc[t] = k_sc[t].view_device(); + } + } + + for (int n = 0; n < surf->nsc; n++) sc_phase(surf->sc[n],SC_PRE); + + upload_surf_collide_models(); +} + +/* ---------------------------------------------------------------------- + re-blit the models and push them to the device + pre_collide(), backup() and restore() all rewrite members of the live + model -- d_particles above all, which a grow reallocates -- so the + device image is stale until this runs again +------------------------------------------------------------------------- */ + +void UpdateKokkos::upload_surf_collide_models() +{ + if (surf->nsc == 0) return; + + int slot[SC_NSTYLE]; + for (int t = 0; t < SC_NSTYLE; t++) slot[t] = 0; + + for (int n = 0; n < surf->nsc; n++) { + const int tag = surf_collide_style_tag(surf->sc[n]); + char *dst = k_sc[tag].view_host().data() + (size_t) slot[tag]*sc_sizeof(tag); + switch (tag) { +#define SC_BLIT(TAG,TYPE) case TAG: sc_blit(dst,surf->sc[n]); break; + SC_FOREACH(SC_BLIT) +#undef SC_BLIT + } + slot[tag]++; + } + + for (int t = 0; t < SC_NSTYLE; t++) { + if (!nsc_style[t]) continue; + k_sc[t].modify_host(); + k_sc[t].sync_device(); + d_sc[t] = k_sc[t].view_device(); + } +} + +/* ---------------------------------------------------------------------- */ + +void UpdateKokkos::grow_tally_computes() +{ + int ncoll = 0, nreact = 0; + + for (int m = 0; m < nsurf_tally; m++) { + if (ComputeSurfCollisionTallyKokkos* c = + dynamic_cast(slist_active[m])) { + c->grow_after_overflow(); + + // growing reallocated the compute's row buffer, so the copy the kernel + // reads still points at the old, too-small one. Without re-blitting + // it the repeated attempt overflows on the same row and the retry + // loop never terminates + +#ifdef SPARTA_KOKKOS_FIXED_LISTS + slist_active_coll_tally_copy[ncoll++].copy(c); +#else + tally_buf_blit(k_slist_coll_tally,ncoll++,c); +#endif + } else if (ComputeSurfReactionTallyKokkos* c = + dynamic_cast(slist_active[m])) { + c->grow_after_overflow(); +#ifdef SPARTA_KOKKOS_FIXED_LISTS + slist_active_react_tally_copy[nreact++].copy(c); +#else + tally_buf_blit(k_slist_react_tally,nreact++,c); +#endif + } + } + +#ifndef SPARTA_KOKKOS_FIXED_LISTS + tally_buf_sync(k_slist_coll_tally,d_slist_coll_tally); + tally_buf_sync(k_slist_react_tally,d_slist_react_tally); +#endif +} + +/* ---------------------------------------------------------------------- + mark (mark=1) or rewind to (mark=0) the append position of every per-event + tally compute + the move kernel runs once per migration iteration and the tally accumulates + across all of them, so a retried attempt must not zero the counter -- it + rewinds to where the current iteration started, discarding only the rows + the aborted attempt appended +------------------------------------------------------------------------- */ + +void UpdateKokkos::rewind_tally_computes(int mark) +{ + for (int m = 0; m < nsurf_tally; m++) { + if (ComputeSurfCollisionTallyKokkos* c = + dynamic_cast(slist_active[m])) + { if (mark) c->mark_ntally(); else c->rewind_ntally(); } + else if (ComputeSurfReactionTallyKokkos* c = + dynamic_cast(slist_active[m])) + { if (mark) c->mark_ntally(); else c->rewind_ntally(); } + } } diff --git a/src/KOKKOS/update_kokkos.h b/src/KOKKOS/update_kokkos.h index e8f1b9391..8d9f998ae 100644 --- a/src/KOKKOS/update_kokkos.h +++ b/src/KOKKOS/update_kokkos.h @@ -31,19 +31,25 @@ #include "surf_collide_td_kokkos.h" #include "surf_collide_cll_kokkos.h" #include "compute_boundary_kokkos.h" +#include "compute_react_boundary_kokkos.h" #include "compute_surf_kokkos.h" +#include "compute_surf_collision_tally_kokkos.h" +#include "compute_surf_reaction_tally_kokkos.h" #include "compute_isurf_grid_kokkos.h" #include "compute_react_isurf_grid_kokkos.h" #include "compute_react_surf_kokkos.h" namespace SPARTA_NS { -#define KOKKOS_MAX_SURF_COLL_PER_TYPE 2 -// 9 supported surf_collide types (specular, diffuse, vanish, piston, -// transparent, adiabatic, impulsive, td, cll) x KOKKOS_MAX_SURF_COLL_PER_TYPE -#define KOKKOS_MAX_TOT_SURF_COLL 18 -#define KOKKOS_MAX_BLIST 2 -#define KOKKOS_MAX_SLIST 2 +// surf_collide style tags, used to dispatch on device where the host's +// virtual SurfCollide::collide() is not available + +enum{SC_SPECULAR,SC_DIFFUSE,SC_VANISH,SC_PISTON,SC_TRANSPARENT, + SC_ADIABATIC,SC_IMPULSIVE,SC_TD,SC_CLL,SC_NSTYLE}; + +// host-side phases every surf_collide model is run through + +enum{SC_PRE,SC_POST,SC_BACKUP,SC_RESTORE}; struct s_UPDATE_REDUCE { // per-step counters are bigint since they can exceed 2^31 @@ -160,24 +166,137 @@ class UpdateKokkos : public Update { KKCopy grid_kk_copy; KKCopy domain_kk_copy; - int sc_type_list[KOKKOS_MAX_TOT_SURF_COLL]; - int sc_map[KOKKOS_MAX_TOT_SURF_COLL]; - KKCopy sc_kk_specular_copy[KOKKOS_MAX_SURF_COLL_PER_TYPE]; - KKCopy sc_kk_diffuse_copy[KOKKOS_MAX_SURF_COLL_PER_TYPE]; - KKCopy sc_kk_vanish_copy[KOKKOS_MAX_SURF_COLL_PER_TYPE]; - KKCopy sc_kk_piston_copy[KOKKOS_MAX_SURF_COLL_PER_TYPE]; - KKCopy sc_kk_transparent_copy[KOKKOS_MAX_SURF_COLL_PER_TYPE]; - KKCopy sc_kk_adiabatic_copy[KOKKOS_MAX_SURF_COLL_PER_TYPE]; - KKCopy sc_kk_impulsive_copy[KOKKOS_MAX_SURF_COLL_PER_TYPE]; - KKCopy sc_kk_td_copy[KOKKOS_MAX_SURF_COLL_PER_TYPE]; - KKCopy sc_kk_cll_copy[KOKKOS_MAX_SURF_COLL_PER_TYPE]; - - //KKCopy blist_active_copy[KOKKOS_MAX_GLIST]; + // surf_collide models used to sit in fixed-size KKCopy arrays here, nine + // styles x two instances. This class is itself the functor handed by + // value to every move kernel, and each model nests its own surf_react + // copies, so those arrays made sizeof(UpdateKokkos) 224 KB -- copied to + // the device on every launch -- while capping a run at two instances of + // each style, which an ordinary model with three wall temperatures hits. + // The models now live in device memory instead, one buffer per style sized + // at run time, and the functor carries only the buffers and two index + // maps. The bytes are blitted in rather than constructed there, which is + // what KKCopy::copy() already does (see kokkos_copy.h) and is sound for + // the same reason: on device the models are only read, through + // KOKKOS_INLINE_FUNCTION members, so the vtable pointer is never used and + // the View handles they carry are kept alive by the originals in surf->sc. + + DAT::t_int_1d d_sc_type; // surf_collide index -> style tag + DAT::t_int_1d d_sc_map; // surf_collide index -> slot in style + DAT::tdual_char_1d k_sc[SC_NSTYLE]; // blitted models, one buffer per style + DAT::t_char_1d d_sc[SC_NSTYLE]; + + int nsc_style[SC_NSTYLE]; // # of instances of each style + + // the surf_collide index maps depend only on surf->sc[n]->style, which is + // fixed for a run (surf_collide is a between-runs command), but + // setup_surf_collide_models() runs once per migration iteration. Build + // them once per run and keep the host mirrors, instead of allocating a + // fresh mirror and re-uploading both maps every iteration. + // init() invalidates, so a new run picks up any style change + + int nsc_index_cached; // surf->nsc the maps were built for + DAT::t_int_1d::host_mirror_type h_sc_type; + DAT::t_int_1d::host_mirror_type h_sc_map; + + static int surf_collide_style_tag(class SurfCollide *); + static size_t sc_sizeof(int); + void sc_phase(class SurfCollide *, int); + void setup_surf_collide_models(); // count, blit and upload, once per move + void upload_surf_collide_models(); // re-blit after backup()/restore() + + // dispatch a surface collision to the model at surf_collide index n + // the nine-way switch is what the host's virtual call becomes on device + + template + KOKKOS_INLINE_FUNCTION + Particle::OnePart* surf_collide_dispatch(const int n, Particle::OnePart *&ip, + double &dtremain, const int isurf, + const double *norm, const int isr, + int &reaction, + const DAT::t_int_scalar &d_retry, + const DAT::t_int_scalar &d_nlocal) const + { + const int m = d_sc_map[n]; + +#define SC_CASE(TAG,TYPE) \ + case TAG: \ + return ((const TYPE *) d_sc[TAG].data())[m]. \ + template collide_kokkos \ + (ip,dtremain,isurf,norm,isr,reaction,d_retry,d_nlocal); + + switch (d_sc_type[n]) { + SC_CASE(SC_SPECULAR,SurfCollideSpecularKokkos) + SC_CASE(SC_DIFFUSE,SurfCollideDiffuseKokkos) + SC_CASE(SC_VANISH,SurfCollideVanishKokkos) + SC_CASE(SC_PISTON,SurfCollidePistonKokkos) + SC_CASE(SC_TRANSPARENT,SurfCollideTransparentKokkos) + SC_CASE(SC_ADIABATIC,SurfCollideAdiabaticKokkos) + SC_CASE(SC_IMPULSIVE,SurfCollideImpulsiveKokkos) + SC_CASE(SC_TD,SurfCollideTDKokkos) + SC_CASE(SC_CLL,SurfCollideCLLKokkos) + } + +#undef SC_CASE + + return NULL; + } + + // the active tally computes, partitioned by type. Two representations, + // selected by SPARTA_KOKKOS_FIXED_LISTS (see kokkos_type.h): + // - default: one runtime-sized device buffer per type, objects blitted in + // exactly as KKCopy::copy() does (kokkos_copy.h:71). No instance cap. + // - SPARTA_KOKKOS_FIXED_LISTS: the original fixed-size KKCopy arrays, held + // by value in this functor, capped at two instances of each type. + // The dispatch sites in the move kernel are written once, against the + // UK_* accessors below, so only these declarations and the setup routine + // differ between the two. + +#ifdef SPARTA_KOKKOS_FIXED_LISTS KKCopy slist_active_copy[KOKKOS_MAX_SLIST]; KKCopy slist_active_isurf_copy[KOKKOS_MAX_SLIST]; + KKCopy slist_active_coll_tally_copy[KOKKOS_MAX_SLIST]; + KKCopy slist_active_react_tally_copy[KOKKOS_MAX_SLIST]; KKCopy slist_active_react_isurf_copy[KOKKOS_MAX_SLIST]; KKCopy slist_active_react_surf_copy[KOKKOS_MAX_SLIST]; KKCopy blist_active_copy[KOKKOS_MAX_BLIST]; + KKCopy blist_active_react_copy[KOKKOS_MAX_BLIST]; + ComputeReactBoundaryKokkos tmp_compute_react_boundary_kk; + + // unused fixed slots must not alias a compute that may be reallocated or + // deleted while they still reference count it + + ComputeBoundaryKokkos tmp_compute_boundary_kk; + ComputeSurfKokkos tmp_compute_surf_kk; + ComputeISurfGridKokkos tmp_compute_isurf_grid_kk; + ComputeReactISurfGridKokkos tmp_compute_react_isurf_grid_kk; + ComputeReactSurfKokkos tmp_compute_react_surf_kk; + +#define UK_SLIST_SURF(m) slist_active_copy[m].obj +#define UK_SLIST_ISURF(m) slist_active_isurf_copy[m].obj +#define UK_SLIST_COLL_TALLY(m) slist_active_coll_tally_copy[m].obj +#define UK_SLIST_REACT_TALLY(m) slist_active_react_tally_copy[m].obj +#define UK_SLIST_REACT_ISURF(m) slist_active_react_isurf_copy[m].obj +#define UK_SLIST_REACT_SURF(m) slist_active_react_surf_copy[m].obj +#define UK_BLIST(m) blist_active_copy[m].obj +#define UK_BLIST_REACT(m) blist_active_react_copy[m].obj + +#else + DAT::tdual_char_1d k_slist_surf, k_slist_isurf, k_slist_coll_tally, + k_slist_react_tally, k_slist_react_isurf, + k_slist_react_surf, k_blist, k_blist_react; + DAT::t_char_1d d_slist_surf, d_slist_isurf, d_slist_coll_tally, + d_slist_react_tally, d_slist_react_isurf, + d_slist_react_surf, d_blist, d_blist_react; + +#define UK_SLIST_SURF(m) ((const ComputeSurfKokkos *) d_slist_surf.data())[m] +#define UK_SLIST_ISURF(m) ((const ComputeISurfGridKokkos *) d_slist_isurf.data())[m] +#define UK_SLIST_COLL_TALLY(m) ((const ComputeSurfCollisionTallyKokkos *) d_slist_coll_tally.data())[m] +#define UK_SLIST_REACT_TALLY(m) ((const ComputeSurfReactionTallyKokkos *) d_slist_react_tally.data())[m] +#define UK_SLIST_REACT_ISURF(m) ((const ComputeReactISurfGridKokkos *) d_slist_react_isurf.data())[m] +#define UK_SLIST_REACT_SURF(m) ((const ComputeReactSurfKokkos *) d_slist_react_surf.data())[m] +#define UK_BLIST(m) ((const ComputeBoundaryKokkos *) d_blist.data())[m] +#define UK_BLIST_REACT(m) ((const ComputeReactBoundaryKokkos *) d_blist_react.data())[m] +#endif // partition of slist_active (set in tally_set): // nslist_surf = # of compute surf style tallies (slist_active_copy) @@ -186,18 +305,20 @@ class UpdateKokkos : public Update { // nslist_surf + nslist_isurf + nslist_react_isurf == nsurf_tally int nslist_surf,nslist_isurf,nslist_react_isurf,nslist_react_surf; + int nslist_coll_tally,nslist_react_tally; + int nblist_boundary,nblist_react; + + // grow every per-event tally compute after an overflowed attempt + void grow_tally_computes(); + void rewind_tally_computes(int); + - ComputeBoundaryKokkos tmp_compute_boundary_kk; - ComputeSurfKokkos tmp_compute_surf_kk; - ComputeISurfGridKokkos tmp_compute_isurf_grid_kk; - ComputeReactISurfGridKokkos tmp_compute_react_isurf_grid_kk; - ComputeReactSurfKokkos tmp_compute_react_surf_kk; // int scalars = flags and view-index counters, must stay int // bigint scalars = per-step statistics counters, can exceed 2^31 // in one step at large per-proc particle counts - typedef Kokkos::DualView tdual_int_7; + typedef Kokkos::DualView tdual_int_7; typedef tdual_int_7::t_dev t_int_7; typedef tdual_int_7::t_host t_host_int_7; t_int_7 d_scalars; @@ -246,7 +367,9 @@ class UpdateKokkos : public Update { HAT::t_int_scalar h_error_flag; DAT::t_int_scalar d_retry; + DAT::t_int_scalar d_tally_overflow; HAT::t_int_scalar h_retry; + HAT::t_int_scalar h_tally_overflow; DAT::t_int_scalar d_nlocal; HAT::t_int_scalar h_nlocal; @@ -259,6 +382,7 @@ class UpdateKokkos : public Update { HAT::t_bigint_1d h_bcmirror; void backup(); + void free_particle_backup(); void restore(); t_particle_1d d_particles_backup; diff --git a/src/PYTHON/python_impl.cpp b/src/PYTHON/python_impl.cpp index 22fd18354..0e32de382 100644 --- a/src/PYTHON/python_impl.cpp +++ b/src/PYTHON/python_impl.cpp @@ -410,14 +410,14 @@ void PythonImpl::invoke_function(int ifunc, char *result, double *dvalue) if (dvalue) *dvalue = (double) PY_INT_AS_LONG(pValue); else { char value[128]; - sprintf(value, BIGINT_FORMAT, (bigint) PY_INT_AS_LONG(pValue)); + snprintf(value,sizeof(value), BIGINT_FORMAT, (bigint) PY_INT_AS_LONG(pValue)); strncpy(result, value, Variable::VALUELENGTH - 1); } } else if (otype == DOUBLE) { if (dvalue) *dvalue = PyFloat_AsDouble(pValue); else { char value[128]; - sprintf(value, "%.15g", PyFloat_AsDouble(pValue)); + snprintf(value,sizeof(value), "%.15g", PyFloat_AsDouble(pValue)); strncpy(result, value, Variable::VALUELENGTH - 1); } } else if (otype == STRING) { @@ -576,7 +576,7 @@ int PythonImpl::create_entry(char *name, int ninput, int noutput, } if (!input->variable->internal_style(ivar)) { char str[128]; - sprintf(str,"Variable %s for python command is invalid style",vname); + snprintf(str,sizeof(str),"Variable %s for python command is invalid style",vname); error->all(FLERR, str); } } else { @@ -599,7 +599,7 @@ int PythonImpl::create_entry(char *name, int ninput, int noutput, } if (!input->variable->internal_style(ivar)) { char str[128]; - sprintf(str,"Variable %s for python command is invalid style",vname); + snprintf(str,sizeof(str),"Variable %s for python command is invalid style",vname); error->all(FLERR, str); } } else { @@ -613,7 +613,7 @@ int PythonImpl::create_entry(char *name, int ninput, int noutput, pfuncs[ifunc].svalue[i] = utils::strdup(istr[i] + 2); } else if (utils::strmatch(istr[i], "^iv_")) { char str[128]; - sprintf(str,"Input argument %s cannot be internal variable with string format",istr[i]); + snprintf(str,sizeof(str),"Input argument %s cannot be internal variable with string format",istr[i]); error->all(FLERR, str); } else { pfuncs[ifunc].ivarflag[i] = 0; diff --git a/src/VTK/dump_grid_vtk.cpp b/src/VTK/dump_grid_vtk.cpp index 9e7c37afc..7249e4ef2 100644 --- a/src/VTK/dump_grid_vtk.cpp +++ b/src/VTK/dump_grid_vtk.cpp @@ -261,7 +261,7 @@ void DumpGridVTK::buf2arrays(int n, double *mybuf) else if (fields[f].type == STRING) { // idstr: buf holds the numeric cell ID (ubuf-encoded); convert to the // hierarchical string form, mirroring DumpGrid::write_text - char str[32]; + char str[128]; // Grid::id_num2str assumes a 128 byte buffer grid->id_num2str((cellint) ubuf(mybuf[c]).i,str); ((vtkStringArray *) paa)->InsertNextValue(str); } else diff --git a/src/adapt_grid.cpp b/src/adapt_grid.cpp index d6df6bd36..c3c826bfd 100644 --- a/src/adapt_grid.cpp +++ b/src/adapt_grid.cpp @@ -420,10 +420,10 @@ void AdaptGrid::process_args(int narg, char **arg) if (maxlevel_request && maxlevel < maxlevel_request && me == 0) { char str[256]; #ifdef SPARTA_BIGBIG - sprintf(str,"Reduced maxlevel for grid adaptation from %d to %d because it induces " + snprintf(str,sizeof(str),"Reduced maxlevel for grid adaptation from %d to %d because it induces " "cell IDs that exceed %d bits",maxlevel_request,maxlevel,(int) sizeof(cellint)*8); #else - sprintf(str,"Reduced maxlevel for grid adaptation from %d to %d because it induces " + snprintf(str,sizeof(str),"Reduced maxlevel for grid adaptation from %d to %d because it induces " "cell IDs that exceed %d bits, compiling with -DSPARTA_BIGBIG may allow further adaptation" ,maxlevel_request,maxlevel,(int) sizeof(cellint)*8); #endif diff --git a/src/collide.cpp b/src/collide.cpp index 35f397605..659f757c7 100644 --- a/src/collide.cpp +++ b/src/collide.cpp @@ -204,7 +204,7 @@ void Collide::init() } if (flag) { char str[128]; - sprintf(str,"%d species do not define correct rotational " + snprintf(str,sizeof(str),"%d species do not define correct rotational " "temps for discrete model",flag); error->all(FLERR,str); } @@ -226,7 +226,7 @@ void Collide::init() } if (flag) { char str[128]; - sprintf(str,"%d species do not define correct vibrational " + snprintf(str,sizeof(str),"%d species do not define correct vibrational " "modes for discrete model",flag); error->all(FLERR,str); } diff --git a/src/collide_vss.cpp b/src/collide_vss.cpp index 735096d16..738097356 100644 --- a/src/collide_vss.cpp +++ b/src/collide_vss.cpp @@ -984,7 +984,7 @@ void CollideVSS::read_param_file(char *fname) FILE *fp = fopen(fname,"r"); if (fp == NULL) { char str[128]; - sprintf(str,"Cannot open VSS parameter file %s",fname); + snprintf(str,sizeof(str),"Cannot open VSS parameter file %s",fname); error->one(FLERR,str); } @@ -1070,7 +1070,7 @@ void CollideVSS::read_param_file(char *fname) if (params[i][i].diam < 0.0) { char str[128]; - sprintf(str,"Species %s did not appear in VSS parameter file", + snprintf(str,sizeof(str),"Species %s did not appear in VSS parameter file", particle->species[i].id); error->one(FLERR,str); } diff --git a/src/compute.cpp b/src/compute.cpp index 6d84775c7..b72798163 100644 --- a/src/compute.cpp +++ b/src/compute.cpp @@ -65,6 +65,22 @@ Compute::Compute(SPARTA *sparta, int narg, char **arg) : Pointers(sparta) kokkos_flag = 0; copy = copymode = 0; + + // per-particle/grid/surf output pointers must start NULL. A subclass only + // assigns the ones it uses, and usually only on its first invocation, so a + // compute that is defined but never consumed would otherwise carry garbage + // here and free() it at teardown. ComputeKEParticleKokkos's destructor + // does exactly that, which corrupts the heap and surfaces much later as an + // abort inside MPI_Finalize. + + vector_particle = NULL; + array_particle = NULL; + vector_grid = NULL; + array_grid = NULL; + vector_surf = NULL; + array_surf = NULL; + vector_surf_tally = NULL; + array_surf_tally = NULL; } /* ---------------------------------------------------------------------- */ diff --git a/src/compute_gas_collision_tally.h b/src/compute_gas_collision_tally.h index 9bc46b916..abfcfe71f 100644 --- a/src/compute_gas_collision_tally.h +++ b/src/compute_gas_collision_tally.h @@ -29,6 +29,7 @@ namespace SPARTA_NS { class ComputeGasCollisionTally : public Compute { public: ComputeGasCollisionTally(class SPARTA *, int, char **); + ComputeGasCollisionTally(class SPARTA* sparta) : Compute(sparta) {} // needed for Kokkos ~ComputeGasCollisionTally(); void compute_per_tally(); void clear(); diff --git a/src/compute_gas_reaction_tally.h b/src/compute_gas_reaction_tally.h index 3b3868d3a..97bb3cd6a 100644 --- a/src/compute_gas_reaction_tally.h +++ b/src/compute_gas_reaction_tally.h @@ -29,6 +29,7 @@ namespace SPARTA_NS { class ComputeGasReactionTally : public Compute { public: ComputeGasReactionTally(class SPARTA *, int, char **); + ComputeGasReactionTally(class SPARTA* sparta) : Compute(sparta) {} // needed for Kokkos ~ComputeGasReactionTally(); void compute_per_tally(); void clear(); diff --git a/src/compute_ke_particle.h b/src/compute_ke_particle.h index a482ea4e9..7f66d0e02 100644 --- a/src/compute_ke_particle.h +++ b/src/compute_ke_particle.h @@ -35,9 +35,8 @@ class ComputeKEParticle : public Compute { protected: int nmax; - - private: - double *ke; + double *ke; // aliased by vector_particle; protected so the Kokkos + // subclass can clear it before ~ComputeKEParticle() frees it }; } diff --git a/src/compute_property_surf.cpp b/src/compute_property_surf.cpp index 738056a90..5abc2b071 100644 --- a/src/compute_property_surf.cpp +++ b/src/compute_property_surf.cpp @@ -135,13 +135,17 @@ void ComputePropertySurf::init() distributed = surf->distributed; - // one-time setup of cglobal list of owned elements in the group - // nsown = # of surf elements I own - // nchoose = # of nown surf elements in surface group - // cglobal[] = global indices for nchoose elements - // used to access lines/tris in Surf - // clocal[] = local indices for nchoose elements - // used to access nown data from per-surf computes,fixes,variables + // one-time setup of cglobal, the local -> global index map for the elements + // this proc owns + // nsown = # of surf elements I own, and the number of output rows: the + // per-surf output is sized nsown below and every consumer reads + // surf->nown rows of it (fix_ave_surf.cpp:215) + // cglobal[] therefore has nsown entries, one per owned element, NOT one per + // group member. The pack methods zero the rows whose element is outside + // the group, which is what their "else buf[n] = 0.0" branch is for; if + // cglobal held only group members those loops would read past it and then + // index lines[]/tris[] with whatever came back + // nchoose = # of owned elements that are in the group, reported only int me = comm->me; int nprocs = comm->nprocs; @@ -160,20 +164,7 @@ void ComputePropertySurf::init() nsown = surf->nown; int m; - nchoose = 0; - for (int i = 0; i < nsown; i++) { - if (dimension == 2) { - if (!distributed) m = me + i*nprocs; - else m = i; - if (lines[m].mask & groupbit) nchoose++; - } else { - if (!distributed) m = me + i*nprocs; - else m = i; - if (tris[m].mask & groupbit) nchoose++; - } - } - - memory->create(cglobal,nchoose,"property/surf:cglobal"); + memory->create(cglobal,nsown,"property/surf:cglobal"); if (nvalues == 1) memory->create(vector_surf,nsown,"property/surf:vector_surf"); else @@ -181,14 +172,13 @@ void ComputePropertySurf::init() nchoose = 0; for (int i = 0; i < nsown; i++) { + if (!distributed) m = me + i*nprocs; + else m = i; + cglobal[i] = m; if (dimension == 2) { - if (!distributed) m = me + i*nprocs; - else m = i; - if (lines[m].mask & groupbit) cglobal[nchoose++] = m; + if (lines[m].mask & groupbit) nchoose++; } else { - if (!distributed) m = me + i*nprocs; - else m = i; - if (tris[m].mask & groupbit) cglobal[nchoose++] = m; + if (tris[m].mask & groupbit) nchoose++; } } } @@ -237,7 +227,7 @@ void ComputePropertySurf::pack_id(int n) Surf::Line *lines; if (distributed) lines = surf->mylines; else lines = surf->lines; - for (int i = 0; i < nchoose; i++) { + for (int i = 0; i < nsown; i++) { m = cglobal[i]; if (lines[m].mask & groupbit) buf[n] = lines[m].id; else buf[n] = 0.0; @@ -247,7 +237,7 @@ void ComputePropertySurf::pack_id(int n) Surf::Tri *tris; if (distributed) tris = surf->mytris; else tris = surf->tris; - for (int i = 0; i < nchoose; i++) { + for (int i = 0; i < nsown; i++) { m = cglobal[i]; if (tris[m].mask & groupbit) buf[n] = tris[m].id; else buf[n] = 0.0; diff --git a/src/compute_react_boundary.cpp b/src/compute_react_boundary.cpp index a0bb7d6da..ad45038ed 100644 --- a/src/compute_react_boundary.cpp +++ b/src/compute_react_boundary.cpp @@ -98,6 +98,8 @@ ComputeReactBoundary(SPARTA *sparta, int narg, char **arg) : ComputeReactBoundary::~ComputeReactBoundary() { + if (copy || copymode) return; + memory->destroy(reaction2col); memory->destroy(array); memory->destroy(myarray); diff --git a/src/compute_react_boundary.h b/src/compute_react_boundary.h index f11b18b4e..3ffeea33f 100644 --- a/src/compute_react_boundary.h +++ b/src/compute_react_boundary.h @@ -29,6 +29,7 @@ namespace SPARTA_NS { class ComputeReactBoundary : public Compute { public: ComputeReactBoundary(class SPARTA *, int, char **); + ComputeReactBoundary(class SPARTA* sparta) : Compute(sparta) {} // needed for Kokkos ~ComputeReactBoundary(); virtual void init(); virtual void compute_array(); diff --git a/src/compute_react_isurf_grid.cpp b/src/compute_react_isurf_grid.cpp index a7eac68a4..fb3ddaa6c 100644 --- a/src/compute_react_isurf_grid.cpp +++ b/src/compute_react_isurf_grid.cpp @@ -159,7 +159,7 @@ void ComputeReactISurfGrid::init() if (flagall && comm->me == 0) { char str[128]; - sprintf(str, + snprintf(str,sizeof(str), "Compute react/isurf/grid " BIGINT_FORMAT " surfs " "are not assigned to surf react model",flagall); error->warning(FLERR,str); diff --git a/src/compute_react_surf.cpp b/src/compute_react_surf.cpp index 8a9219c14..bba1b4952 100644 --- a/src/compute_react_surf.cpp +++ b/src/compute_react_surf.cpp @@ -151,7 +151,7 @@ void ComputeReactSurf::init() if (flagall && comm->me == 0) { char str[128]; - sprintf(str, + snprintf(str,sizeof(str), "Compute react/surf " BIGINT_FORMAT " surfs are not assigned to surf react model",flagall); error->warning(FLERR,str); diff --git a/src/compute_surf_collision_tally.h b/src/compute_surf_collision_tally.h index 9280e7a5b..6f7f4ccab 100644 --- a/src/compute_surf_collision_tally.h +++ b/src/compute_surf_collision_tally.h @@ -29,10 +29,11 @@ namespace SPARTA_NS { class ComputeSurfCollisionTally : public Compute { public: ComputeSurfCollisionTally(class SPARTA *, int, char **); + ComputeSurfCollisionTally(class SPARTA* sparta) : Compute(sparta) {} // needed for Kokkos ~ComputeSurfCollisionTally(); void init(); void compute_per_tally(); - void clear(); + virtual void clear(); void surf_tally(double, int, int, int, Particle::OnePart *, Particle::OnePart *, Particle::OnePart *); int tallyinfo(surfint *&); diff --git a/src/compute_surf_reaction_tally.h b/src/compute_surf_reaction_tally.h index 8c035fff3..8e4562fb3 100644 --- a/src/compute_surf_reaction_tally.h +++ b/src/compute_surf_reaction_tally.h @@ -29,10 +29,11 @@ namespace SPARTA_NS { class ComputeSurfReactionTally : public Compute { public: ComputeSurfReactionTally(class SPARTA *, int, char **); + ComputeSurfReactionTally(class SPARTA* sparta) : Compute(sparta) {} // needed for Kokkos ~ComputeSurfReactionTally(); void init(); void compute_per_tally(); - void clear(); + virtual void clear(); void surf_tally(double, int, int, int, Particle::OnePart *, Particle::OnePart *, Particle::OnePart *); int tallyinfo(surfint *&); diff --git a/src/create_grid.cpp b/src/create_grid.cpp index a61510d10..b4d646abd 100644 --- a/src/create_grid.cpp +++ b/src/create_grid.cpp @@ -242,7 +242,7 @@ void CreateGrid::command(int narg, char **arg) int nbits = plevels[nlevels-1].nbits + plevels[nlevels-1].newbits; if (nbits > sizeof(cellint)*8) { char str[128]; - sprintf(str,"Hierarchical grid induces cell IDs that exceed %d bits", + snprintf(str,sizeof(str),"Hierarchical grid induces cell IDs that exceed %d bits", (int) sizeof(cellint)*8); error->all(FLERR,str); } diff --git a/src/create_isurf.cpp b/src/create_isurf.cpp index eaef5b200..85ce0cf76 100644 --- a/src/create_isurf.cpp +++ b/src/create_isurf.cpp @@ -293,7 +293,7 @@ void CreateISurf::set_corners() MPI_Allreduce(&ofull,&allofull,1,MPI_INT,MPI_SUM,world); if (allofull) { char str[128]; - sprintf(str, + snprintf(str,sizeof(str), "Create_isurf could not determine whether some corner \ values are inside or outside with respect to the surface"); error->all(FLERR,str); @@ -384,7 +384,7 @@ void CreateISurf::set_multi() MPI_Allreduce(&ofull,&allofull,1,MPI_INT,MPI_SUM,world); if (allofull) { char str[128]; - sprintf(str, + snprintf(str,sizeof(str), "Create_isurf could not determine whether some corner \ values are inside or outside with respect to the surface"); error->all(FLERR,str); diff --git a/src/create_particles.cpp b/src/create_particles.cpp index fd46cc8ff..8a91145dd 100644 --- a/src/create_particles.cpp +++ b/src/create_particles.cpp @@ -439,7 +439,7 @@ void CreateParticles::command(int narg, char **arg) MPI_Allreduce(&nme,&nglobal,1,MPI_SPARTA_BIGINT,MPI_SUM,world); if (!region && !nrho_var_flag && nglobal-nprevious != np) { char str[128]; - sprintf(str,"Created unexpected # of particles: " + snprintf(str,sizeof(str),"Created unexpected # of particles: " BIGINT_FORMAT " versus " BIGINT_FORMAT, nglobal-nprevious,np); if (comm->me == 0) error->warning(FLERR,str); diff --git a/src/custom.cpp b/src/custom.cpp index 29cacb929..09ce87384 100644 --- a/src/custom.cpp +++ b/src/custom.cpp @@ -1416,7 +1416,7 @@ void Custom::read_coarse_files(char *fname, int numfile, int colcount) if (count_all) { char str[128]; - sprintf(str,"%d coarse grid points are outside simulation box",count_all); + snprintf(str,sizeof(str),"%d coarse grid points are outside simulation box",count_all); error->all(FLERR,str); } diff --git a/src/cut3d.cpp b/src/cut3d.cpp index a2565e6aa..a427a581d 100644 --- a/src/cut3d.cpp +++ b/src/cut3d.cpp @@ -894,7 +894,7 @@ void Cut3d::clip_tris() /* if (id == VERBOSE_ID) { char str[24]; - sprintf(str,"Partial FACE %d %d\n",iface,ivert); + snprintf(str,sizeof(str),"Partial FACE %d %d\n",iface,ivert); print_bpg(str); } */ @@ -946,7 +946,7 @@ void Cut3d::clip_tris() /* if (id == VERBOSE_ID) { char str[24]; - sprintf(str,"After FACE %d\n",iface); + snprintf(str,sizeof(str),"After FACE %d\n",iface); print_bpg(str); } */ diff --git a/src/dump.cpp b/src/dump.cpp index 2b27c34a9..88e5b280a 100644 --- a/src/dump.cpp +++ b/src/dump.cpp @@ -420,7 +420,7 @@ void Dump::openfile() if (compressed) { #ifdef SPARTA_GZIP char gzip[128]; - sprintf(gzip,"gzip -6 > %s",filecurrent); + snprintf(gzip,sizeof(gzip),"gzip -6 > %s",filecurrent); #ifdef _WIN32 fp = _popen(gzip,"wb"); #else @@ -644,7 +644,7 @@ void Dump::modify_params(int narg, char **arg) error->all(FLERR, "Dump_modify int format does not contain d character"); char str[8]; - sprintf(str,"%s",BIGINT_FORMAT); + snprintf(str,sizeof(str),"%s",BIGINT_FORMAT); *ptr = '\0'; sprintf(format_bigint_user,"%s%s%s",format_int_user,&str[1],ptr+1); *ptr = 'd'; diff --git a/src/fix_ave_histo.cpp b/src/fix_ave_histo.cpp index 7d774be0e..39f274a29 100644 --- a/src/fix_ave_histo.cpp +++ b/src/fix_ave_histo.cpp @@ -1017,7 +1017,7 @@ void FixAveHisto::options(int iarg, int narg, char **arg) fp = fopen(arg[iarg+1],"w"); if (fp == NULL) { char str[128]; - sprintf(str,"Cannot open fix ave/histo file %s",arg[iarg+1]); + snprintf(str,sizeof(str),"Cannot open fix ave/histo file %s",arg[iarg+1]); error->one(FLERR,str); } } diff --git a/src/fix_ave_time.cpp b/src/fix_ave_time.cpp index 0abea6fb0..984206e56 100644 --- a/src/fix_ave_time.cpp +++ b/src/fix_ave_time.cpp @@ -23,6 +23,7 @@ #include "variable.h" #include "memory.h" #include "error.h" +#include "sparta_masks.h" using namespace SPARTA_NS; @@ -312,6 +313,13 @@ FixAveTime::FixAveTime(SPARTA *sparta, int narg, char **arg) : nvalid = nextvalid(); modify->addstep_compute_all(nvalid); + + // this fix reduces global scalars/vectors and never writes particle data, + // so the Kokkos wrapper need not push particles back to the device + // afterwards. datamask_read stays ALL_MASK: the inputs may be host + // computes that read particle->particles + + datamask_modify = EMPTY_MASK; } /* ---------------------------------------------------------------------- */ @@ -717,7 +725,7 @@ void FixAveTime::options(int iarg, int narg, char **arg) fp = fopen(arg[iarg+1],"w"); if (fp == NULL) { char str[128]; - sprintf(str,"Cannot open fix ave/time file %s",arg[iarg+1]); + snprintf(str,sizeof(str),"Cannot open fix ave/time file %s",arg[iarg+1]); error->one(FLERR,str); } } diff --git a/src/fix_controller.cpp b/src/fix_controller.cpp index c02c8fd4b..2a7078f96 100644 --- a/src/fix_controller.cpp +++ b/src/fix_controller.cpp @@ -28,6 +28,7 @@ #include "modify.h" #include "update.h" #include "variable.h" +#include "sparta_masks.h" using namespace SPARTA_NS; @@ -136,6 +137,12 @@ FixController::FixController(SPARTA *sparta, int narg, char **arg) : control = input->variable->compute_equal(ivariable); firsttime = 1; + + // this fix reads one global value and sets an internal variable; it never + // writes particle data, so the Kokkos wrapper need not push particles + // back to the device afterwards + + datamask_modify = EMPTY_MASK; } /* ---------------------------------------------------------------------- */ diff --git a/src/fix_emit.cpp b/src/fix_emit.cpp index e957b01c3..1502ced40 100644 --- a/src/fix_emit.cpp +++ b/src/fix_emit.cpp @@ -188,7 +188,7 @@ int FixEmit::subsonic_temperature_check(int flag, double tempmax) MPI_Allreduce(&tempmax,&allmax,1,MPI_DOUBLE,MPI_MAX,world); if (comm->me == 0) { char str[128]; - sprintf(str,"Excessive subsonic thermal temp = %g",allmax); + snprintf(str,sizeof(str),"Excessive subsonic thermal temp = %g",allmax); error->warning(FLERR,str); } return 1; diff --git a/src/fix_emit_face_file.cpp b/src/fix_emit_face_file.cpp index cb9d2af18..d86ec60fa 100644 --- a/src/fix_emit_face_file.cpp +++ b/src/fix_emit_face_file.cpp @@ -77,6 +77,7 @@ FixEmitFaceFile::FixEmitFaceFile(SPARTA *sparta, int narg, char **arg) : // optional args frac_user = 1.0; + twopass = 0; int iarg = 6; options(narg-iarg,&arg[iarg]); @@ -131,6 +132,12 @@ FixEmitFaceFile::FixEmitFaceFile(SPARTA *sparta, int narg, char **arg) : FixEmitFaceFile::~FixEmitFaceFile() { + // a Kokkos functor copy must not free the file mesh or the task arrays; + // every sibling has this guard (fix_emit_face.cpp:117, fix_emit.cpp:63, + // fix.cpp:74) and this class was missing it + + if (copymode) return; + delete [] mesh.which; delete [] mesh.imesh; delete [] mesh.jmesh; @@ -344,6 +351,18 @@ void FixEmitFaceFile::create_task(int icell) ------------------------------------------------------------------------- */ void FixEmitFaceFile::perform_task() +{ + if (!twopass) perform_task_onepass(); + else perform_task_twopass(); +} + +/* ---------------------------------------------------------------------- + perform insertion in one pass thru tasks + this is simpler, somewhat faster code + but uses random #s differently than Kokkos, so insertions are different +------------------------------------------------------------------------- */ + +void FixEmitFaceFile::perform_task_onepass() { int pcell,ninsert,nactual,isp,ispecies,id; double temp_thermal,temp_rot,temp_vib; @@ -504,6 +523,194 @@ void FixEmitFaceFile::perform_task() } } +/* ---------------------------------------------------------------------- + perform insertion the way Kokkos does in two passes thru tasks + this uses random #s the same as Kokkos, for easier debugging +------------------------------------------------------------------------- */ + +void FixEmitFaceFile::perform_task_twopass() +{ + int pcell,ninsert,nactual,isp,ispecies,id; + double temp_thermal,temp_rot,temp_vib; + double indot,scosine,rn,ntarget,vr; + double beta_un,normalized_distbn_fn,theta,erot,evib; + double x[3],v[3]; + double *lo,*hi,*vstream,*cummulative,*vscale; + Particle::OnePart *p; + + double dt = update->dt; + int *species = particle->mixture[imix]->species; + + // if subsonic, re-compute particle inflow counts for each task + // also computes current temp_thermal and vstream in insertion cells + + if (subsonic) subsonic_inflow(); + + // insert particles for each task = cell + // ntarget/ninsert is either perspecies or for all species + // for one particle: + // x = random position on subset of face that overlaps with file grid + // v = randomized thermal velocity + vstream + // first stage: normal dimension (ndim) + // second stage: parallel dimensions (pdim1,pdim2) + + // double while loop until randomized particle velocity meets 2 criteria + // inner do-while loop: + // v = vstream-component + vthermal is into simulation box + // see Bird 1994, p 425 + // outer do-while loop: + // shift Maxwellian distribution by stream velocity component + // see Bird 1994, p 259, eq 12.5 + + int nfix_update_custom = modify->n_update_custom; + + // first pass: draw every task's insertion count, before any particle is + // generated. This is the ordering the Kokkos version necessarily + // produces -- its scan must know all counts before candidate arrays can + // be sized -- so the two agree only when this pass is used. + // Mirrors FixEmitFace::perform_task_twopass() (fix_emit_face.cpp:675) + + int ninsert_dim1 = perspecies ? nspecies : 1; + int **ninsert_values; + memory->create(ninsert_values,ntask,ninsert_dim1,"fix_emit_face_file:ninsert"); + + for (int i = 0; i < ntask; i++) { + if (perspecies) { + for (isp = 0; isp < nspecies; isp++) { + ntarget = tasks[i].ntargetsp[isp]+random->uniform(); + ninsert_values[i][isp] = static_cast (ntarget); + } + } else { + ntarget = tasks[i].ntarget+random->uniform(); + ninsert_values[i][0] = static_cast (ntarget); + } + } + + for (int i = 0; i < ntask; i++) { + pcell = tasks[i].pcell; + lo = tasks[i].lo; + hi = tasks[i].hi; + + temp_thermal = tasks[i].temp_thermal; + temp_rot = tasks[i].temp_rot; + temp_vib = tasks[i].temp_vib; + vscale = tasks[i].vscale; + vstream = tasks[i].vstream; + + indot = vstream[0]*normal[0] + vstream[1]*normal[1] + vstream[2]*normal[2]; + + if (perspecies) { + for (isp = 0; isp < nspecies; isp++) { + ispecies = species[isp]; + ninsert = ninsert_values[i][isp]; + scosine = indot / vscale[isp]; + + nactual = 0; + for (int m = 0; m < ninsert; m++) { + x[0] = lo[0] + random->uniform() * (hi[0]-lo[0]); + if (domain->axisymmetric) + x[1] = sqrt(lo[1]*lo[1] + + random->uniform() * (hi[1]*hi[1]-lo[1]*lo[1])); + else x[1] = lo[1] + random->uniform() * (hi[1]-lo[1]); + if (dimension == 3) x[2] = lo[2] + random->uniform() * (hi[2]-lo[2]); + else x[2] = 0.0; + + if (region && !region->match(x)) continue; + + do { + do beta_un = (6.0*random->uniform() - 3.0); + while (beta_un + scosine < 0.0); + normalized_distbn_fn = 2.0 * (beta_un + scosine) / + (scosine + sqrt(scosine*scosine + 2.0)) * + exp(0.5 + (0.5*scosine)*(scosine-sqrt(scosine*scosine + 2.0)) - + beta_un*beta_un); + } while (normalized_distbn_fn < random->uniform()); + + v[ndim] = beta_un*vscale[isp]*normal[ndim] + vstream[ndim]; + + theta = MY_2PI * random->uniform(); + vr = vscale[isp] * sqrt(-log(random->uniform())); + v[pdim] = vr * sin(theta) + vstream[pdim]; + v[qdim] = vr * cos(theta) + vstream[qdim]; + erot = particle->erot(ispecies,temp_rot,random); + evib = particle->evib(ispecies,temp_vib,random); + id = MAXSMALLINT*random->uniform(); + + particle->add_particle(id,ispecies,pcell,x,v,erot,evib); + nactual++; + + p = &particle->particles[particle->nlocal-1]; + p->flag = PINSERT; + p->dtremain = dt * random->uniform(); + + if (nfix_update_custom) + modify->update_custom(particle->nlocal-1,temp_thermal, + temp_rot,temp_vib,vstream); + } + + nsingle += nactual; + } + + } else { + cummulative = tasks[i].cummulative; + ninsert = ninsert_values[i][0]; + + nactual = 0; + for (int m = 0; m < ninsert; m++) { + rn = random->uniform(); + isp = 0; + while (cummulative[isp] < rn) isp++; + ispecies = species[isp]; + scosine = indot / vscale[isp]; + + x[0] = lo[0] + random->uniform() * (hi[0]-lo[0]); + if (domain->axisymmetric) + x[1] = sqrt(lo[1]*lo[1] + + random->uniform() * (hi[1]*hi[1]-lo[1]*lo[1])); + else x[1] = lo[1] + random->uniform() * (hi[1]-lo[1]); + if (dimension == 3) x[2] = lo[2] + random->uniform() * (hi[2]-lo[2]); + else x[2] = 0.0; + + if (region && !region->match(x)) continue; + + do { + do beta_un = (6.0*random->uniform() - 3.0); + while (beta_un + scosine < 0.0); + normalized_distbn_fn = 2.0 * (beta_un + scosine) / + (scosine + sqrt(scosine*scosine + 2.0)) * + exp(0.5 + (0.5*scosine)*(scosine-sqrt(scosine*scosine + 2.0)) - + beta_un*beta_un); + } while (normalized_distbn_fn < random->uniform()); + + v[ndim] = beta_un*vscale[isp]*normal[ndim] + vstream[ndim]; + + theta = MY_2PI * random->uniform(); + vr = vscale[isp] * sqrt(-log(random->uniform())); + v[pdim] = vr * sin(theta) + vstream[pdim]; + v[qdim] = vr * cos(theta) + vstream[qdim]; + erot = particle->erot(ispecies,temp_rot,random); + evib = particle->evib(ispecies,temp_vib,random); + id = MAXSMALLINT*random->uniform(); + + particle->add_particle(id,ispecies,pcell,x,v,erot,evib); + nactual++; + + p = &particle->particles[particle->nlocal-1]; + p->flag = PINSERT; + p->dtremain = dt * random->uniform(); + + if (nfix_update_custom) + modify->update_custom(particle->nlocal-1,temp_thermal, + temp_rot,temp_vib,vstream); + } + + nsingle += nactual; + } + } + + memory->destroy(ninsert_values); +} + /* ---------------------------------------------------------------------- scan file for section-ID, read regular grid of values into Mesh data struct only called by proc 0 @@ -522,7 +729,7 @@ void FixEmitFaceFile::read_file(char *file, char *section) FILE *fp = fopen(file,"r"); if (fp == NULL) { char str[128]; - sprintf(str,"Cannot open inflow file %s",file); + snprintf(str,sizeof(str),"Cannot open inflow file %s",file); error->one(FLERR,str); } @@ -1303,7 +1510,12 @@ int FixEmitFaceFile::option(int narg, char **arg) return 2; } - error->all(FLERR,"Illegal fix emit/face command"); + if (strcmp(arg[0],"twopass") == 0) { + twopass = 1; + return 1; + } + + error->all(FLERR,"Illegal fix emit/face/file command"); return 0; } diff --git a/src/fix_emit_face_file.h b/src/fix_emit_face_file.h index 2ba0ccb34..4e6d140fe 100644 --- a/src/fix_emit_face_file.h +++ b/src/fix_emit_face_file.h @@ -30,12 +30,37 @@ namespace SPARTA_NS { class FixEmitFaceFile : public FixEmit { public: FixEmitFaceFile(class SPARTA *, int, char **); - ~FixEmitFaceFile(); - void init(); + ~FixEmitFaceFile() override; + void init() override; - private: + // one insertion task for a cell and a face + + struct Task { + double lo[3]; // lower-left corner of overlap of cell/file + double hi[3]; // upper-right corner of overlap of cell/file + double area; // area of face + double ntarget; // # of mols to insert for all species + double *ntargetsp; // # of mols to insert for each species, + // only defined for PERSPECIES + + int icell; // associated cell index, unsplit or split cell + int pcell; // associated cell index for particles + // unsplit or sub cell (not split cell) + + // interpolated file values or defaults from mixture params + + double nrho; + double temp_thermal,temp_rot,temp_vib; + double press; + double vstream[3]; + double *fraction; + double *cummulative; + double *vscale; + }; + + protected: int imix,iface,subsonic,subsonic_style,subsonic_warning; - int npertask,nthresh; + int npertask,nthresh,twopass; double frac_user; double tprefactor,soundspeed_mixture; @@ -71,30 +96,6 @@ class FixEmitFaceFile : public FixEmit { Mesh mesh; - // one insertion task for a cell and a face - - struct Task { - double lo[3]; // lower-left corner of overlap of cell/file - double hi[3]; // upper-right corner of overlap of cell/file - double area; // area of face - double ntarget; // # of mols to insert for all species - double *ntargetsp; // # of mols to insert for each species, - // only defined for PERSPECIES - - int icell; // associated cell index, unsplit or split cell - int pcell; // associated cell index for particles - // unsplit or sub cell (not split cell) - - // interpolated file values or defaults from mixture params - - double nrho; - double temp_thermal,temp_rot,temp_vib; - double press; - double vstream[3]; - double *fraction; - double *cummulative; - double *vscale; - }; // ntask = # of tasks is stored by parent class Task *tasks; // list of particle insertion tasks @@ -121,13 +122,15 @@ class FixEmitFaceFile : public FixEmit { int split(int); - void subsonic_inflow(); - void subsonic_sort(); - void subsonic_grid(); + virtual void subsonic_inflow(); + virtual void subsonic_sort(); + virtual void subsonic_grid(); - void create_task(int); - void perform_task(); - void grow_task(); + virtual void create_task(int); + virtual void perform_task(); + void perform_task_onepass(); + virtual void perform_task_twopass(); + virtual void grow_task(); int option(int, char **); void print_task(int); diff --git a/src/fix_grid_check.cpp b/src/fix_grid_check.cpp index e4c29ce96..cfc95ddbf 100644 --- a/src/fix_grid_check.cpp +++ b/src/fix_grid_check.cpp @@ -118,7 +118,7 @@ void FixGridCheck::end_of_step() if (icell < 0 || icell >= nglocal) { if (outflag == ERROR) { char str[128]; - sprintf(str, + snprintf(str,sizeof(str), "Particle %d,%d on proc %d is in invalid cell index %d" " on timestep " BIGINT_FORMAT, i,particles[i].id,comm->me,icell,update->ntimestep); @@ -142,7 +142,7 @@ void FixGridCheck::end_of_step() // i,icell,cells[icell].id,x[0],x[1],x[2], // lo[0],lo[1],lo[2],hi[0],hi[1],hi[2]); char str[128]; - sprintf(str, + snprintf(str,sizeof(str), "Particle %d,%d on proc %d is outside cell " CELLINT_FORMAT " on timestep " BIGINT_FORMAT, i,particles[i].id,comm->me,cells[icell].id, @@ -157,7 +157,7 @@ void FixGridCheck::end_of_step() if (cells[icell].nsplit > 1) { if (outflag == ERROR) { char str[128]; - sprintf(str, + snprintf(str,sizeof(str), "Particle %d,%d on proc %d is in split cell " CELLINT_FORMAT " on timestep " BIGINT_FORMAT, i,particles[i].id,comm->me,cells[icell].id, @@ -172,7 +172,7 @@ void FixGridCheck::end_of_step() if (cinfo[icell].type == INSIDE) { if (outflag == ERROR) { char str[128]; - sprintf(str, + snprintf(str,sizeof(str), "Particle %d,%d on proc %d is in interior cell " CELLINT_FORMAT " on timestep " BIGINT_FORMAT, i,particles[i].id,comm->me,cells[icell].id,update->ntimestep); @@ -186,7 +186,7 @@ void FixGridCheck::end_of_step() if (cinfo[icell].volume == 0.0) { if (outflag == ERROR) { char str[128]; - sprintf(str, + snprintf(str,sizeof(str), "Particle %d,%d on proc %d is in volume=0 cell " CELLINT_FORMAT " on timestep " BIGINT_FORMAT, i,particles[i].id,comm->me,cells[icell].id,update->ntimestep); @@ -218,7 +218,7 @@ void FixGridCheck::end_of_step() if (!pflag) { if (outflag == ERROR) { char str[128]; - sprintf(str, + snprintf(str,sizeof(str), "Particle %d,%d on proc %d at %g %g %g is inside surfs in cell " CELLINT_FORMAT " on timestep " BIGINT_FORMAT, i,particles[i].id,comm->me,x[0],x[1],x[2],cells[icell].id, @@ -239,7 +239,7 @@ void FixGridCheck::end_of_step() if (subcell != icell) { if (outflag == ERROR) { char str[128]; - sprintf(str, + snprintf(str,sizeof(str), "Particle %d,%d on proc %d is in wrong sub cell %d not %d" " on timestep " BIGINT_FORMAT, i,particles[i].id,comm->me,icell,subcell, @@ -260,21 +260,21 @@ void FixGridCheck::end_of_step() MPI_Allreduce(&nflag,&all,1,MPI_SPARTA_BIGINT,MPI_SUM,world); if (all && comm->me == 0) { char str[128]; - sprintf(str,BIGINT_FORMAT " particles in wrong cells on timestep " + snprintf(str,sizeof(str),BIGINT_FORMAT " particles in wrong cells on timestep " BIGINT_FORMAT,all,update->ntimestep); error->warning(FLERR,str); } MPI_Allreduce(&nflag_surf,&all,1,MPI_SPARTA_BIGINT,MPI_SUM,world); if (all && comm->me == 0) { char str[128]; - sprintf(str,BIGINT_FORMAT " particles inside surfs on timestep " + snprintf(str,sizeof(str),BIGINT_FORMAT " particles inside surfs on timestep " BIGINT_FORMAT,all,update->ntimestep); error->warning(FLERR,str); } MPI_Allreduce(&nflag_split,&all,1,MPI_SPARTA_BIGINT,MPI_SUM,world); if (all && comm->me == 0) { char str[128]; - sprintf(str,BIGINT_FORMAT " particles in wrong sub cells on timestep " + snprintf(str,sizeof(str),BIGINT_FORMAT " particles in wrong sub cells on timestep " BIGINT_FORMAT,all,update->ntimestep); error->warning(FLERR,str); } diff --git a/src/fix_halt.cpp b/src/fix_halt.cpp index 767672887..f5c605f9b 100644 --- a/src/fix_halt.cpp +++ b/src/fix_halt.cpp @@ -59,7 +59,7 @@ FixHalt::FixHalt(SPARTA *sparta, int narg, char **arg) : } else { if (!utils::strmatch(arg[iarg],"^v_")) { char msg[128]; - sprintf(msg, "Invalid fix halt attribute %s", arg[iarg]); + snprintf(msg,sizeof(msg), "Invalid fix halt attribute %s", arg[iarg]); error->all(FLERR, msg); } @@ -102,7 +102,7 @@ FixHalt::FixHalt(SPARTA *sparta, int narg, char **arg) : else if (strcmp(arg[iarg + 1], "continue") == 0) eflag = CONTINUE; else { char msg[128]; - sprintf(msg, "Unknown fix halt error condition %s", arg[iarg]); + snprintf(msg,sizeof(msg), "Unknown fix halt error condition %s", arg[iarg]); error->all(FLERR, msg); } iarg += 2; @@ -112,7 +112,7 @@ FixHalt::FixHalt(SPARTA *sparta, int narg, char **arg) : iarg += 2; } else { char msg[128]; - sprintf(msg, "Unknown fix halt keyword %s", arg[iarg]); + snprintf(msg,sizeof(msg), "Unknown fix halt keyword %s", arg[iarg]); error->all(FLERR, msg); } } @@ -172,11 +172,11 @@ void FixHalt::init() ivar = input->variable->find(idvar); char msg[128]; if (ivar < 0) { - sprintf(msg, "Could not find fix halt variable %s", idvar); + snprintf(msg,sizeof(msg), "Could not find fix halt variable %s", idvar); error->all(FLERR, msg); } if (input->variable->equal_style(ivar) == 0) { - sprintf(msg, "Fix halt variable %s is not equal-style variable", idvar); + snprintf(msg,sizeof(msg), "Fix halt variable %s is not equal-style variable", idvar); error->all(FLERR, msg); } } @@ -237,7 +237,7 @@ void FixHalt::end_of_step() // print message with ID of fix halt in case multiple instances char message[128]; - sprintf(message, "Fix halt condition for fix-id %s met on step " BIGINT_FORMAT " with value %g", + snprintf(message,sizeof(message), "Fix halt condition for fix-id %s met on step " BIGINT_FORMAT " with value %g", id, update->ntimestep, attvalue); if (eflag == HARD) { error->all(FLERR, message); diff --git a/src/fix_print.cpp b/src/fix_print.cpp index 19d0febf0..eb89f2753 100644 --- a/src/fix_print.cpp +++ b/src/fix_print.cpp @@ -21,6 +21,7 @@ #include "variable.h" #include "memory.h" #include "error.h" +#include "sparta_masks.h" using namespace SPARTA_NS; @@ -89,6 +90,13 @@ FixPrint::FixPrint(SPARTA *sparta, int narg, char **arg) : } delete [] title; + + // printing touches no particle data, so the Kokkos wrapper around this fix + // need not push particles back to the device afterwards. datamask_read + // stays ALL_MASK: the format string may contain an equal-style variable + // that references a host compute over particles + + datamask_modify = EMPTY_MASK; } /* ---------------------------------------------------------------------- */ diff --git a/src/grid.cpp b/src/grid.cpp index 0c3c722fb..f3749e371 100644 --- a/src/grid.cpp +++ b/src/grid.cpp @@ -1504,7 +1504,7 @@ void Grid::find_neighbors() if (nall) { char str[128]; - sprintf(str,"Owned cells with unknown neighbors = %d",nall); + snprintf(str,sizeof(str),"Owned cells with unknown neighbors = %d",nall); error->all(FLERR,str); } } @@ -2107,7 +2107,7 @@ void Grid::type_check(int outflag) if (unknownall) { char str[128]; - sprintf(str,"Grid cells marked as unknown = %d",unknownall); + snprintf(str,sizeof(str),"Grid cells marked as unknown = %d",unknownall); error->all(FLERR,str); } @@ -2158,7 +2158,7 @@ void Grid::type_check(int outflag) MPI_Allreduce(&inside,&insideall,1,MPI_INT,MPI_SUM,world); if (insideall) { char str[128]; - sprintf(str,"Grid cell interior corner points marked as unknown " + snprintf(str,sizeof(str),"Grid cell interior corner points marked as unknown " "(volume will be wrong if cell is effectively outside) = %d", insideall); if (comm->me == 0) error->warning(FLERR,str); @@ -2168,7 +2168,7 @@ void Grid::type_check(int outflag) MPI_Allreduce(&outside,&outsideall,1,MPI_INT,MPI_SUM,world); if (outsideall) { char str[128]; - sprintf(str,"Grid cell corner points on boundary marked as unknown = %d", + snprintf(str,sizeof(str),"Grid cell corner points on boundary marked as unknown = %d", outsideall); error->all(FLERR,str); } @@ -2182,7 +2182,7 @@ void Grid::type_check(int outflag) MPI_Allreduce(&volzero,&volzeroall,1,MPI_INT,MPI_SUM,world); if (outsideall) { char str[128]; - sprintf(str,"Grid cells marked outside, but with zero volume = %d", + snprintf(str,sizeof(str),"Grid cells marked outside, but with zero volume = %d", volzeroall); error->all(FLERR,str); } @@ -2660,7 +2660,7 @@ int Grid::check_uniform_group(int igroup, int *nxyz, MPI_Allreduce(&sflag,&allsflag,1,MPI_INT,MPI_SUM,world); if (allsflag && surf->implicit) { char str[128]; - sprintf(str, + snprintf(str,sizeof(str), "Read_isurfs adding surfs to %d cells which already have surfs", allsflag); error->all(FLERR,str); diff --git a/src/grid.h b/src/grid.h index 6ed8fb46e..601d0d523 100644 --- a/src/grid.h +++ b/src/grid.h @@ -289,7 +289,7 @@ class Grid : protected Pointers { bigint pack_restart(char *); bigint unpack_restart(char *); - bigint memory_usage(); + virtual bigint memory_usage(); void debug(); diff --git a/src/grid_id.cpp b/src/grid_id.cpp index 55a7f03a5..76ae2bfa2 100644 --- a/src/grid_id.cpp +++ b/src/grid_id.cpp @@ -541,7 +541,10 @@ void Grid::id_num2str(cellint id, char *str) newbits = plevels[level].newbits; mask = (1L << newbits) - 1; ichild = id & mask; - if (offset >= 110) break; // prevent buffer overflow + // the previous iteration overwrote the terminating NUL with '-', so the + // bail-out must re-terminate or the caller reads past the last write + + if (offset >= 110) { str[offset] = '\0'; break; } // prevent buffer overflow snprintf(&str[offset], 128 - offset, CELLINT_FORMAT, ichild); offset = strlen(str); id = id >> newbits; diff --git a/src/input.cpp b/src/input.cpp index a1fd48c1f..86962f952 100644 --- a/src/input.cpp +++ b/src/input.cpp @@ -518,7 +518,7 @@ void Input::substitute(char *&str, char *&str2, int &max, int &max2, int flag) if (var[i] == '\0') error->one(FLERR,"Invalid immediate variable"); var[i] = '\0'; beyond = ptr + strlen(var) + 3; - sprintf(immediate,"%.20g",variable->compute_equal(var)); + snprintf(immediate,sizeof(immediate),"%.20g",variable->compute_equal(var)); value = immediate; // single character variable name, e.g. $a @@ -759,7 +759,7 @@ int Input::expand_args(int narg, char **arg, int mode, char **&earg) if (expandflag < 0) { char str[256]; - sprintf(str,"Cannot use wildcard with %s because it " + snprintf(str,sizeof(str),"Cannot use wildcard with %s because it " "does not produce multiple values",arg[iarg]); error->all(FLERR,str); } @@ -899,7 +899,7 @@ int Input::execute_command() if (sparta->suffix_enable && sparta->suffix) { char command2[256]; - sprintf(command2,"%s/%s",command,sparta->suffix); + snprintf(command2,sizeof(command2),"%s/%s",command,sparta->suffix); if (0) return 0; // dummy line to enable else-if macro expansion #define COMMAND_CLASS @@ -1398,7 +1398,7 @@ void Input::collide_command() if (sparta->suffix_enable) { if (sparta->suffix) { char estyle[256]; - sprintf(estyle,"%s/%s",arg[0],sparta->suffix); + snprintf(estyle,sizeof(estyle),"%s/%s",arg[0],sparta->suffix); if (0) return; @@ -1532,7 +1532,7 @@ void Input::react_command() if (sparta->suffix_enable) { if (sparta->suffix) { char estyle[256]; - sprintf(estyle,"%s/%s",arg[0],sparta->suffix); + snprintf(estyle,sizeof(estyle),"%s/%s",arg[0],sparta->suffix); if (0) return; diff --git a/src/memory.cpp b/src/memory.cpp index 1677045fb..a17d7f99a 100644 --- a/src/memory.cpp +++ b/src/memory.cpp @@ -58,7 +58,7 @@ void *Memory::smalloc(bigint nbytes, const char *name, int align) if (ptr == NULL) { char str[128]; - sprintf(str,"Failed to allocate " BIGINT_FORMAT " bytes for array %s", + snprintf(str,sizeof(str),"Failed to allocate " BIGINT_FORMAT " bytes for array %s", nbytes,name); error->one(FLERR,str); } @@ -96,7 +96,7 @@ void *Memory::srealloc(void *ptr, bigint nbytes, const char *name, int align) if (ptr == NULL) { char str[128]; - sprintf(str,"Failed to reallocate " BIGINT_FORMAT " bytes for array %s", + snprintf(str,sizeof(str),"Failed to reallocate " BIGINT_FORMAT " bytes for array %s", nbytes,name); error->one(FLERR,str); } @@ -120,6 +120,6 @@ void Memory::sfree(void *ptr) void Memory::fail(const char *name) { char str[128]; - sprintf(str,"Cannot create/grow a vector/array of pointers for %s",name); + snprintf(str,sizeof(str),"Cannot create/grow a vector/array of pointers for %s",name); error->one(FLERR,str); } diff --git a/src/output.cpp b/src/output.cpp index 88410ea1e..4fc873f26 100644 --- a/src/output.cpp +++ b/src/output.cpp @@ -297,10 +297,15 @@ void Output::write(bigint ntimestep) if (next_restart == ntimestep) { if (next_restart_single == ntimestep) { - char *file = new char[strlen(restart1) + 16]; + // the length must be taken before *ptr = '\0' truncates restart1 at + // the wildcard. taken after, it is the prefix length, not the buffer + // size, and snprintf silently drops the tail of the name + + int nfile = strlen(restart1) + 16; + char *file = new char[nfile]; char *ptr = strchr(restart1,'*'); *ptr = '\0'; - snprintf(file,strlen(restart1)+16,"%s" BIGINT_FORMAT "%s",restart1,ntimestep,ptr+1); + snprintf(file,nfile,"%s" BIGINT_FORMAT "%s",restart1,ntimestep,ptr+1); *ptr = '*'; if (last_restart != ntimestep) restart->write(file); delete [] file; @@ -383,10 +388,14 @@ void Output::write_dump(bigint ntimestep) void Output::write_restart(bigint ntimestep) { if (restart_flag_single) { - char *file = new char[strlen(restart1) + 16]; + // see the note in Output::write(): the length must be taken before the + // wildcard is truncated away + + int nfile = strlen(restart1) + 16; + char *file = new char[nfile]; char *ptr = strchr(restart1,'*'); *ptr = '\0'; - snprintf(file,strlen(restart1)+16,"%s" BIGINT_FORMAT "%s",restart1,ntimestep,ptr+1); + snprintf(file,nfile,"%s" BIGINT_FORMAT "%s",restart1,ntimestep,ptr+1); *ptr = '*'; restart->write(file); delete [] file; diff --git a/src/particle.cpp b/src/particle.cpp index 3556d5afc..b7a97f874 100644 --- a/src/particle.cpp +++ b/src/particle.cpp @@ -171,7 +171,7 @@ void Particle::init() if (species[isp].vibdof <= 2) continue; if (species[isp].vibdiscrete_read == 0) { char str[128]; - sprintf(str,"Discrete vibrational info for species %s not read in", + snprintf(str,sizeof(str),"Discrete vibrational info for species %s not read in", species[isp].id); error->all(FLERR,str); } diff --git a/src/particle.h b/src/particle.h index a77d2446a..206ce036d 100644 --- a/src/particle.h +++ b/src/particle.h @@ -186,7 +186,7 @@ class Particle : protected Pointers { virtual void pack_custom(int, char *); virtual void unpack_custom(char *, int); - bigint memory_usage(); + virtual bigint memory_usage(); protected: int me; diff --git a/src/react_bird.cpp b/src/react_bird.cpp index b2728600c..d86fdd5e1 100644 --- a/src/react_bird.cpp +++ b/src/react_bird.cpp @@ -579,7 +579,7 @@ void ReactBird::readfile(char *fname) fp = fopen(fname,"r"); if (fp == NULL) { char str[128]; - sprintf(str,"Cannot open reaction file %s",fname); + snprintf(str,sizeof(str),"Cannot open reaction file %s",fname); error->one(FLERR,str); } } diff --git a/src/read_grid.cpp b/src/read_grid.cpp index b7d1be74a..17d9bba9d 100644 --- a/src/read_grid.cpp +++ b/src/read_grid.cpp @@ -534,7 +534,7 @@ void ReadGrid::header() int nbits = plevels[nlevels-1].nbits + plevels[nlevels-1].newbits; if (nbits > sizeof(cellint)*8) { char str[128]; - sprintf(str,"Hierarchical grid induces cell IDs that exceed %d bits", + snprintf(str,sizeof(str),"Hierarchical grid induces cell IDs that exceed %d bits", (int) sizeof(cellint)*8); error->all(FLERR,str); } diff --git a/src/read_particles.cpp b/src/read_particles.cpp index d79bf485f..8efbc0444 100644 --- a/src/read_particles.cpp +++ b/src/read_particles.cpp @@ -130,7 +130,7 @@ void ReadParticles::command(int narg, char **arg) MPI_Allreduce(&flagme,&flagall,1,MPI_SPARTA_BIGINT,MPI_SUM,world); if (flagall) { char str[128]; - sprintf(str,BIGINT_FORMAT " read-in particles have invalid species", + snprintf(str,sizeof(str),BIGINT_FORMAT " read-in particles have invalid species", flagall); error->all(FLERR,str); } @@ -147,7 +147,7 @@ void ReadParticles::command(int narg, char **arg) MPI_Allreduce(&flagme,&flagall,1,MPI_SPARTA_BIGINT,MPI_SUM,world); if (flagall) { char str[128]; - sprintf(str,BIGINT_FORMAT " read-in particles are inside surface", + snprintf(str,sizeof(str),BIGINT_FORMAT " read-in particles are inside surface", flagall); error->all(FLERR,str); } diff --git a/src/read_restart.cpp b/src/read_restart.cpp index c77c73458..4fcf2b6f1 100644 --- a/src/read_restart.cpp +++ b/src/read_restart.cpp @@ -462,9 +462,15 @@ void ReadRestart::file_search(char *infile, char *outfile) // create outfile with maxint substituted for "*" // use original infile, not pattern, since need to retain "%" in filename + // outfile is the caller's buffer, sized new char[strlen(arg[0]) + 16] with + // arg[0] == infile, so that is the bound. it must be computed before + // *ptr = '\0' truncates infile at the wildcard: taken after, it is the + // prefix length and snprintf silently drops the tail of the name + + size_t nout = strlen(infile) + 16; ptr = strchr(infile,'*'); *ptr = '\0'; - snprintf(outfile,strlen(infile)+16,"%s" BIGINT_FORMAT "%s",infile,maxnum,ptr+1); + snprintf(outfile,nout,"%s" BIGINT_FORMAT "%s",infile,maxnum,ptr+1); *ptr = '*'; // clean up @@ -689,7 +695,7 @@ void ReadRestart::grid_params() int nbits = grid->plevels[maxlevel-1].nbits + grid->plevels[maxlevel-1].newbits; if (nbits > sizeof(cellint)*8) { char str[128]; - sprintf(str,"Hierarchical grid induces cell IDs that exceed %d bits", + snprintf(str,sizeof(str),"Hierarchical grid induces cell IDs that exceed %d bits", (int) sizeof(cellint)*8); error->all(FLERR,str); } diff --git a/src/read_surf.cpp b/src/read_surf.cpp index 405a9dfa5..fc4ee94ff 100644 --- a/src/read_surf.cpp +++ b/src/read_surf.cpp @@ -1996,13 +1996,13 @@ void ReadSurf::check_bounds() if (sminall != 1) { char str[128]; - sprintf(str,"Read_surf minimum surface ID is " BIGINT_FORMAT,sminall); + snprintf(str,sizeof(str),"Read_surf minimum surface ID is " BIGINT_FORMAT,sminall); error->all(FLERR,str); } if (smaxall != nsurf_all) { char str[128]; - sprintf(str,"Read_surf maximum surface ID is " BIGINT_FORMAT,smaxall); + snprintf(str,sizeof(str),"Read_surf maximum surface ID is " BIGINT_FORMAT,smaxall); error->all(FLERR,str); } } @@ -2140,13 +2140,13 @@ void ReadSurf::check_neighbor_norm_2d() if (nerror) { char str[128]; - sprintf(str,"Surface check failed with %d " + snprintf(str,sizeof(str),"Surface check failed with %d " "infinitely thin line pairs",nerror); error->all(FLERR,str); } if (nwarn) { char str[128]; - sprintf(str,"Surface check found %d " + snprintf(str,sizeof(str),"Surface check found %d " "nearly infinitely thin line pairs",nwarn); if (me == 0) error->warning(FLERR,str); } @@ -2223,13 +2223,13 @@ void ReadSurf::check_neighbor_norm_3d() if (nerror) { char str[128]; - sprintf(str,"Surface check failed with %d " + snprintf(str,sizeof(str),"Surface check failed with %d " "infinitely thin triangle pairs",nerror); error->all(FLERR,str); } if (nwarn) { char str[128]; - sprintf(str,"Surface check found %d " + snprintf(str,sizeof(str),"Surface check found %d " "nearly infinitely thin triangle pairs",nwarn); if (me == 0) error->warning(FLERR,str); } @@ -2339,9 +2339,15 @@ void ReadSurf::file_search(char *infile, char *outfile) // create outfile with maxint substituted for "*" // use original infile, not pattern, since need to retain "%" in filename + // outfile is the caller's buffer, sized new char[strlen(arg[0]) + 16] with + // arg[0] == infile, so that is the bound. it must be computed before + // *ptr = '\0' truncates infile at the wildcard: taken after, it is the + // prefix length and snprintf silently drops the tail of the name + + size_t nout = strlen(infile) + 16; ptr = strchr(infile,'*'); *ptr = '\0'; - snprintf(outfile,strlen(infile)+16,"%s" BIGINT_FORMAT "%s",infile,maxnum,ptr+1); + snprintf(outfile,nout,"%s" BIGINT_FORMAT "%s",infile,maxnum,ptr+1); *ptr = '*'; // clean up @@ -2531,7 +2537,7 @@ void ReadSurf::check_point_pairs() if (nbad) { char str[128]; - sprintf(str,"%d read_surf point pairs are too close",nbad); + snprintf(str,sizeof(str),"%d read_surf point pairs are too close",nbad); error->all(FLERR,str); } @@ -2584,7 +2590,7 @@ void ReadSurf::check_point_pairs() if (nbad) { char str[128]; - sprintf(str,"%d read_surf point pairs are too close",nbad); + snprintf(str,sizeof(str),"%d read_surf point pairs are too close",nbad); error->all(FLERR,str); } diff --git a/src/region_intersect.h b/src/region_intersect.h index 1d8861ac6..c4c7ca690 100644 --- a/src/region_intersect.h +++ b/src/region_intersect.h @@ -32,7 +32,7 @@ class RegIntersect : public Region { void init(); int inside(double *); - private: + protected: // Kokkos subclasses flatten these to the device int nregion; int *list; }; diff --git a/src/region_union.h b/src/region_union.h index 49c67972a..8ee17e666 100644 --- a/src/region_union.h +++ b/src/region_union.h @@ -31,7 +31,7 @@ class RegUnion : public Region { ~RegUnion(); int inside(double *); - private: + protected: // Kokkos subclasses flatten these to the device int nregion; int *list; }; diff --git a/src/sparta.cpp b/src/sparta.cpp index 3521bfbd3..d6673fcd9 100644 --- a/src/sparta.cpp +++ b/src/sparta.cpp @@ -282,7 +282,7 @@ SPARTA::SPARTA(int narg, char **arg, MPI_Comm communicator) if (partscreenflag == 0) if (screenflag == 0) { char str[32]; - sprintf(str,"screen.%d",universe->iworld); + snprintf(str,sizeof(str),"screen.%d",universe->iworld); screen = fopen(str,"w"); if (screen == NULL) error->one(FLERR,"Cannot open screen file"); } else if (strcmp(arg[screenflag],"none") == 0) @@ -306,7 +306,7 @@ SPARTA::SPARTA(int narg, char **arg, MPI_Comm communicator) if (partlogflag == 0) if (logflag == 0) { char str[32]; - sprintf(str,"log.sparta.%d",universe->iworld); + snprintf(str,sizeof(str),"log.sparta.%d",universe->iworld); logfile = fopen(str,"w"); if (logfile == NULL) error->one(FLERR,"Cannot open logfile"); } else if (strcmp(arg[logflag],"none") == 0) diff --git a/src/sparta_masks.h b/src/sparta_masks.h index de1b1cdf0..e7f682fa9 100644 --- a/src/sparta_masks.h +++ b/src/sparta_masks.h @@ -45,5 +45,12 @@ #define LINE_MASK 0x00000800 #define TRI_MASK 0x00001000 +// surf, explicit distributed: the owned subset (mylines/mytris) +// separate from LINE_MASK/TRI_MASK because they are separate DualViews +// with their own sync state, and grow_own() resizes only these + +#define MYLINE_MASK 0x00002000 +#define MYTRI_MASK 0x00004000 + #endif diff --git a/src/stats.cpp b/src/stats.cpp index 79e8df09c..2b86b7851 100644 --- a/src/stats.cpp +++ b/src/stats.cpp @@ -459,7 +459,7 @@ void Stats::modify_params(int narg, char **arg) error->all(FLERR, "Stats_modify int format does not contain d character"); char str[8]; - sprintf(str,"%s",BIGINT_FORMAT); + snprintf(str,sizeof(str),"%s",BIGINT_FORMAT); *ptr = '\0'; sprintf(format_bigint_user,"%s%s%s",format_int_user,&str[1],ptr+1); *ptr = 'd'; diff --git a/src/surf.cpp b/src/surf.cpp index fed157ca7..1bc836d6a 100644 --- a/src/surf.cpp +++ b/src/surf.cpp @@ -314,7 +314,7 @@ void Surf::init() if (allflag) { char str[64]; - sprintf(str,BIGINT_FORMAT + snprintf(str,sizeof(str),BIGINT_FORMAT " surface elements with invalid type <= 0",allflag); error->all(FLERR,str); } @@ -338,7 +338,7 @@ void Surf::init() if (allflag) { char str[64]; - sprintf(str,BIGINT_FORMAT + snprintf(str,sizeof(str),BIGINT_FORMAT " surface elements not assigned to a collision model",allflag); error->all(FLERR,str); } @@ -363,7 +363,7 @@ void Surf::init() if (allflag) { char str[128]; - sprintf(str,BIGINT_FORMAT " surface elements with reaction model, " + snprintf(str,sizeof(str),BIGINT_FORMAT " surface elements with reaction model, " "but invalid collision model",allflag); error->all(FLERR,str); } @@ -392,7 +392,7 @@ void Surf::init() if (allflag) { char str[128]; - sprintf(str,BIGINT_FORMAT " transparent surface elements " + snprintf(str,sizeof(str),BIGINT_FORMAT " transparent surface elements " "with invalid collision model or reaction model",allflag); error->all(FLERR,str); } @@ -677,7 +677,7 @@ void Surf::add_surfs(int replace, int ncount, MPI_Allreduce(&flag,&flagall,1,MPI_INT,MPI_SUM,world); if (flagall) { char str[128]; - sprintf(str,"Missing read_surf IDs = %d",flagall); + snprintf(str,sizeof(str),"Missing read_surf IDs = %d",flagall); error->all(FLERR,str); } } @@ -1141,7 +1141,7 @@ void Surf::check_watertight_2d_all() if (ndup) { char str[128]; - sprintf(str,"Watertight check failed with %d duplicate points",ndup); + snprintf(str,sizeof(str),"Watertight check failed with %d duplicate points",ndup); error->all(FLERR,str); } @@ -1164,7 +1164,7 @@ void Surf::check_watertight_2d_all() if (nbad) { char str[128]; - sprintf(str,"Watertight check failed with %d unmatched points",nbad); + snprintf(str,sizeof(str),"Watertight check failed with %d unmatched points",nbad); error->all(FLERR,str); } } @@ -1276,7 +1276,7 @@ int Surf::rendezvous_watertight_2d(int n, char *inbuf, int &flag, int *&proclist MPI_Allreduce(&ndup,&alldup,1,MPI_INT,MPI_SUM,world); if (alldup) { char str[128]; - sprintf(str,"Watertight check failed with %d duplicate points",alldup); + snprintf(str,sizeof(str),"Watertight check failed with %d duplicate points",alldup); error->all(FLERR,str); } @@ -1301,7 +1301,7 @@ int Surf::rendezvous_watertight_2d(int n, char *inbuf, int &flag, int *&proclist MPI_Allreduce(&nbad,&allbad,1,MPI_INT,MPI_SUM,world); if (allbad) { char str[128]; - sprintf(str,"Watertight check failed with %d unmatched points",allbad); + snprintf(str,sizeof(str),"Watertight check failed with %d unmatched points",allbad); error->all(FLERR,str); } @@ -1395,7 +1395,7 @@ void Surf::check_watertight_3d_all() if (ndup) { char str[128]; - sprintf(str,"Watertight check failed with %d duplicate edges",ndup); + snprintf(str,sizeof(str),"Watertight check failed with %d duplicate edges",ndup); error->all(FLERR,str); } @@ -1416,7 +1416,7 @@ void Surf::check_watertight_3d_all() if (nbad) { char str[128]; - sprintf(str,"Watertight check failed with %d unmatched edges",nbad); + snprintf(str,sizeof(str),"Watertight check failed with %d unmatched edges",nbad); error->all(FLERR,str); } } @@ -1579,7 +1579,7 @@ int Surf::rendezvous_watertight_3d(int n, char *inbuf, int &flag, int *&proclist alldup /= 2; // avoid double counting if (alldup) { char str[128]; - sprintf(str,"Watertight check failed with %d duplicate edges",alldup); + snprintf(str,sizeof(str),"Watertight check failed with %d duplicate edges",alldup); error->all(FLERR,str); } @@ -1603,7 +1603,7 @@ int Surf::rendezvous_watertight_3d(int n, char *inbuf, int &flag, int *&proclist allbad /= 2; // avoid double counting if (allbad) { char str[128]; - sprintf(str,"Watertight check failed with %d unmatched edges",allbad); + snprintf(str,sizeof(str),"Watertight check failed with %d unmatched edges",allbad); error->all(FLERR,str); } @@ -1684,7 +1684,7 @@ void Surf::check_point_inside(int old) if (nbadall) { char str[128]; - sprintf(str,"%d surface points are not inside simulation box", + snprintf(str,sizeof(str),"%d surface points are not inside simulation box", nbadall); error->all(FLERR,str); } @@ -1741,14 +1741,14 @@ void Surf::check_point_near_surf_2d() MPI_Allreduce(&nerror,&all,1,MPI_INT,MPI_SUM,world); if (all) { char str[128]; - sprintf(str,"Surface check failed with %d points on lines",all); + snprintf(str,sizeof(str),"Surface check failed with %d points on lines",all); error->all(FLERR,str); } MPI_Allreduce(&nwarn,&all,1,MPI_INT,MPI_SUM,world); if (all) { char str[128]; - sprintf(str,"Surface check found %d points nearly on lines",all); + snprintf(str,sizeof(str),"Surface check found %d points nearly on lines",all); if (comm->me == 0) error->warning(FLERR,str); } } @@ -1810,14 +1810,14 @@ void Surf::check_point_near_surf_3d() MPI_Allreduce(&nerror,&all,1,MPI_INT,MPI_SUM,world); if (all) { char str[128]; - sprintf(str,"Surface check failed with %d points on triangles",all); + snprintf(str,sizeof(str),"Surface check failed with %d points on triangles",all); error->all(FLERR,str); } MPI_Allreduce(&nwarn,&all,1,MPI_INT,MPI_SUM,world); if (all) { char str[128]; - sprintf(str,"Surface check found %d points nearly on triangles",all); + snprintf(str,sizeof(str),"Surface check found %d points nearly on triangles",all); if (comm->me == 0) error->warning(FLERR,str); } } diff --git a/src/surf.h b/src/surf.h index 475b70d12..e553b965f 100644 --- a/src/surf.h +++ b/src/surf.h @@ -245,7 +245,7 @@ class Surf : protected Pointers { virtual void grow(int); virtual void grow_own(int); - bigint memory_usage(); + virtual bigint memory_usage(); // surf_collate.cpp // including callback functions diff --git a/src/surf_react_adsorb.cpp b/src/surf_react_adsorb.cpp index bc894999f..62ab551de 100644 --- a/src/surf_react_adsorb.cpp +++ b/src/surf_react_adsorb.cpp @@ -1743,7 +1743,7 @@ void SurfReactAdsorb::readfile_gs(char *fname) fp = fopen(fname,"r"); if (fp == NULL) { char str[128]; - sprintf(str,"Cannot open reaction file %s",fname); + snprintf(str,sizeof(str),"Cannot open reaction file %s",fname); error->one(FLERR,str); } } @@ -2515,7 +2515,7 @@ void SurfReactAdsorb::readfile_ps(char *fname) fp = fopen(fname,"r"); if (fp == NULL) { char str[128]; - sprintf(str,"Cannot open reaction file %s",fname); + snprintf(str,sizeof(str),"Cannot open reaction file %s",fname); error->one(FLERR,str); } } diff --git a/src/update.cpp b/src/update.cpp index f210546d3..9f120e231 100644 --- a/src/update.cpp +++ b/src/update.cpp @@ -1569,7 +1569,7 @@ template < int DIM, int SURF, int OPT > void Update::move() if (particles[i].flag != PDISCARD) { if (cells[icell].proc == me) { char str[128]; - sprintf(str, + snprintf(str,sizeof(str), "Particle %d on proc %d being sent to self " "on step " BIGINT_FORMAT, i,me,update->ntimestep); diff --git a/src/utils.cpp b/src/utils.cpp index a1a156388..a9b451021 100644 --- a/src/utils.cpp +++ b/src/utils.cpp @@ -217,7 +217,7 @@ double utils::numeric(const char *file, int line, const std::string &str, bool d double rv = 0; char msg[128]; - sprintf(msg,"Floating point number %s in input script or data file is invalid", buf.c_str()); + snprintf(msg,sizeof(msg),"Floating point number %s in input script or data file is invalid", buf.c_str()); try { std::size_t endpos; rv = std::stod(buf, &endpos); @@ -233,7 +233,7 @@ double utils::numeric(const char *file, int line, const std::string &str, bool d else sparta->error->all(file, line, msg); } catch (std::out_of_range const &) { - sprintf(msg,"Floating point number %s in input script or data file is out of range", buf.c_str()); + snprintf(msg,sizeof(msg),"Floating point number %s in input script or data file is out of range", buf.c_str()); if (do_abort) sparta->error->one(file, line, msg); else @@ -285,7 +285,7 @@ int utils::inumeric(const char *file, int line, const std::string &str, bool do_ int rv = 0; char msg[128]; - sprintf(msg,"Integer %s in input script or data file is invalid", buf.c_str()); + snprintf(msg,sizeof(msg),"Integer %s in input script or data file is invalid", buf.c_str()); try { std::size_t endpos; rv = std::stoi(buf, &endpos); @@ -302,7 +302,7 @@ int utils::inumeric(const char *file, int line, const std::string &str, bool do_ sparta->error->all(file, line, msg); } catch (std::out_of_range const &) { char msg[128]; - sprintf(msg,"Integer %s in input script or data file is out of range", buf.c_str()); + snprintf(msg,sizeof(msg),"Integer %s in input script or data file is out of range", buf.c_str()); if (do_abort) sparta->error->one(file, line, msg); else @@ -355,7 +355,7 @@ bigint utils::bnumeric(const char *file, int line, const std::string &str, bool long long rv = 0; char msg[128]; - sprintf(msg,"Integer %s in input script or data file is invalid", buf.c_str()); + snprintf(msg,sizeof(msg),"Integer %s in input script or data file is invalid", buf.c_str()); try { std::size_t endpos; rv = std::stoll(buf, &endpos); @@ -373,7 +373,7 @@ bigint utils::bnumeric(const char *file, int line, const std::string &str, bool sparta->error->all(file, line, msg); } catch (std::out_of_range const &) { char msg[128]; - sprintf(msg,"Integer %s in input script or data file is out of range", buf.c_str()); + snprintf(msg,sizeof(msg),"Integer %s in input script or data file is out of range", buf.c_str()); if (do_abort) sparta->error->one(file, line, msg); else diff --git a/src/variable.cpp b/src/variable.cpp index 6024ef7c0..4218551f8 100644 --- a/src/variable.cpp +++ b/src/variable.cpp @@ -205,7 +205,7 @@ void Variable::set(int narg, char **arg) if (nlast <= 0) error->all(FLERR,"Illegal variable command"); if (narg == 4 && strcmp(arg[3],"pad") == 0) { char digits[12]; - sprintf(digits,"%d",nlast); + snprintf(digits,sizeof(digits),"%d",nlast); pad[nvar] = strlen(digits); } else pad[nvar] = 0; } else if (narg == 4 || (narg == 5 && strcmp(arg[4],"pad") == 0)) { @@ -215,7 +215,7 @@ void Variable::set(int narg, char **arg) error->all(FLERR,"Illegal variable command"); if (narg == 5 && strcmp(arg[4],"pad") == 0) { char digits[12]; - sprintf(digits,"%d",nlast); + snprintf(digits,sizeof(digits),"%d",nlast); pad[nvar] = strlen(digits); } else pad[nvar] = 0; } else error->all(FLERR,"Illegal variable command"); @@ -269,7 +269,7 @@ void Variable::set(int narg, char **arg) data[nvar][0] = NULL; if (narg == 4) { char digits[12]; - sprintf(digits,"%d",num[nvar]); + snprintf(digits,sizeof(digits),"%d",num[nvar]); pad[nvar] = strlen(digits); } else pad[nvar] = 0; } @@ -722,11 +722,11 @@ char *Variable::retrieve(char *name) } else if (style[ivar] == LOOP || style[ivar] == ULOOP) { char result[16]; - if (pad[ivar] == 0) sprintf(result,"%d",which[ivar]+1); + if (pad[ivar] == 0) snprintf(result,sizeof(result),"%d",which[ivar]+1); else { char padstr[16]; - sprintf(padstr,"%%0%dd",pad[ivar]); - sprintf(result,padstr,which[ivar]+1); + snprintf(padstr,sizeof(padstr),"%%0%dd",pad[ivar]); + snprintf(result,sizeof(result),padstr,which[ivar]+1); } int n = strlen(result) + 1; delete [] data[ivar][0]; @@ -962,7 +962,7 @@ void Variable::internal_create(char *name, double value) { if (find(name) >= 0) { char str[128]; - sprintf(str,"Creation of internal-style variable %s which already exists", name); + snprintf(str,sizeof(str),"Creation of internal-style variable %s which already exists", name); error->all(FLERR,str); } @@ -977,7 +977,7 @@ void Variable::internal_create(char *name, double value) if (!utils::is_id(name)) { char str[128]; - sprintf(str,"Variable name %s must have only letters, numbers, or underscores", name); + snprintf(str,sizeof(str),"Variable name %s must have only letters, numbers, or underscores", name); error->all(FLERR,str); } @@ -3246,7 +3246,7 @@ int Variable::int_between_brackets(char *&ptr, int varallow, const char *caller) while (*ptr && *ptr != ']') { if (!isdigit(*ptr)) { char str[128]; - sprintf(str,"Non digit character between brackets in %s",caller); + snprintf(str,sizeof(str),"Non digit character between brackets in %s",caller); error->all(FLERR,str); } ptr++; @@ -3255,12 +3255,12 @@ int Variable::int_between_brackets(char *&ptr, int varallow, const char *caller) if (*ptr != ']') { char str[128]; - sprintf(str,"Mismatched brackets in %s",caller); + snprintf(str,sizeof(str),"Mismatched brackets in %s",caller); error->all(FLERR,str); } if (ptr == start) { char str[128]; - sprintf(str,"Empty brackets in %s",caller); + snprintf(str,sizeof(str),"Empty brackets in %s",caller); error->all(FLERR,str); } @@ -3289,7 +3289,7 @@ int Variable::int_between_brackets(char *&ptr, int varallow, const char *caller) if (index == 0) { char str[128]; - sprintf(str,"Index between brackets must be positive in %s",caller); + snprintf(str,sizeof(str),"Index between brackets must be positive in %s",caller); error->all(FLERR,str); } return index; @@ -4617,7 +4617,7 @@ VarReader::VarReader(SPARTA *sparta, char *, char *file, int flag) : fp = fopen(file,"r"); if (fp == NULL) { char str[128]; - sprintf(str,"Cannot open file variable file %s",file); + snprintf(str,sizeof(str),"Cannot open file variable file %s",file); error->one(FLERR,str); } } else fp = NULL;