Skip to content

WIP: CPU coupling with stork for interpolation #33

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
MattRolchigo wants to merge 1 commit into
base: main
Choose a base branch
from
Open

WIP: CPU coupling with stork for interpolation #33

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions applications/SingleLayer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,8 @@ void run( int argc, char* argv[] )
Finch::Layer app( db, grid );
app.run( exec_space(), db, grid, beam, fd );

// Write the temperature data used by ExaCA/other post-processing
app.writeSolidificationData( db.sampling, grid.getComm() );
// Get the solidification data from the simulation and write to file
app.getSolidificationData( grid, MPI_COMM_WORLD, db.sampling, true );
app.getLowerSolidificationDataBounds( grid.getComm() );
app.getUpperSolidificationDataBounds( grid.getComm() );
}
Expand Down
3 changes: 3 additions & 0 deletions cmake/FinchConfig.cmakein
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,7 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}" )
list(APPEND CMAKE_PREFIX_PATH @CMAKE_PREFIX_PATH@)
include("${CMAKE_CURRENT_LIST_DIR}/Finch_Targets.cmake")
find_dependency(Cabana REQUIRED COMPONENTS Cabana::Grid)
if(Finch_ENABLE_STORK)
find_dependency(Stork REQUIRED)
endif()
find_dependency(nlohmann_json REQUIRED)
9 changes: 8 additions & 1 deletion src/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,19 @@
configure_file(Finch_Core_Config.hpp.cmakein Finch_Core_Config.hpp)

find_package(Stork REQUIRED)
Comment thread
streeve marked this conversation as resolved.
file(GLOB CORE_HEADERS GLOB *.hpp)
file(GLOB CORE_BEAM_HEADERS MovingBeam/*.hpp)
file(GLOB CORE_SOURCE *.cpp MovingBeam/*.cpp)
# Gather all .cpp files in the include/ directory
file(GLOB_RECURSE SRC_FILES include/*.cpp)

# Gather all .h/.hpp files in the include/ directory
file(GLOB_RECURSE INCLUDE_FILES include/*.hpp)

Comment thread
streeve marked this conversation as resolved.
add_library(Core ${CORE_SOURCE})
add_library(Finch::Core ALIAS Core)

target_link_libraries(Core Cabana::Grid nlohmann_json::nlohmann_json)
target_link_libraries(Core Cabana::Grid nlohmann_json::nlohmann_json Stork::Stork)
Comment thread
streeve marked this conversation as resolved.

target_include_directories(Core PUBLIC
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
Expand Down
27 changes: 24 additions & 3 deletions src/Finch_Grid.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ class Grid
Cabana::Grid::Array<double, entity_type, mesh_type, memory_space>;
using view_type = typename array_type::view_type;

int comm_rank, comm_size;
int comm_rank, comm_size, num_points_x, num_points_y, num_points_z;

// Construct from components
Grid( MPI_Comm comm, const double cell_size,
Expand Down Expand Up @@ -135,8 +135,29 @@ class Grid
// Note: this is an entirely separate array on purpose (no shallow copy)
T0 = Cabana::Grid::createArray<double, memory_space>( name, layout );

// create halo
halo = createHalo( Cabana::Grid::FaceHaloPattern<3>(), halo_width, *T );
// create halo - requires second nearest neighbor comm for temperature
// data sampling on nodes
halo = createHalo( Cabana::Grid::NodeHaloPattern<3>(), halo_width, *T );

// Size in x, y, z for each MPI rank, not including ghost nodes
auto local_mesh = getLocalMesh();
num_points_x =
std::round( ( local_mesh.highCorner( Cabana::Grid::Own(), 0 ) -
local_mesh.lowCorner( Cabana::Grid::Own(), 0 ) ) /
cell_size ) +
1;
num_points_y =
std::round( ( local_mesh.highCorner( Cabana::Grid::Own(), 1 ) -
local_mesh.lowCorner( Cabana::Grid::Own(), 1 ) ) /
cell_size ) +
1;
num_points_z =
std::round( ( local_mesh.highCorner( Cabana::Grid::Own(), 2 ) -
local_mesh.lowCorner( Cabana::Grid::Own(), 2 ) ) /
cell_size ) +
1;
Info << "Points per direction: " << num_points_x << ", " << num_points_y
<< ", " << num_points_z << std::endl;
}

auto getLocalMesh()
Expand Down
34 changes: 34 additions & 0 deletions src/Finch_Inputs.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,9 @@ struct Sampling
std::string format;
std::string directory_name = "solidification";
bool enabled;
int fine_factor = 1;
std::array<double, 3> global_low_corner;
std::array<double, 3> global_high_corner;
};

struct TimeMonitor
Expand Down Expand Up @@ -471,6 +474,29 @@ class Inputs
if ( db.contains( "sampling" ) )
{
const std::string sampling_type = db["sampling"]["type"];
// Refinement factor for heat transport to solidification grids
// (defaults to 1)
if ( db["sampling"].contains( "fine_factor" ) )
sampling.fine_factor = db["sampling"]["fine_factor"];
else
sampling.fine_factor = 1;
#ifndef Finch_ENABLE_STORK
if ( sampling.fine_factor != 1 )
throw std::runtime_error( "Error: Finch must be compiled with "
"Stork to perform interpolation" );
#endif
// Region used for solidification data collection (defaults to
// global domain bounds if not given)
if ( db["sampling"].contains( "global_low_corner" ) )
sampling.global_low_corner =
db["sampling"]["global_low_corner"];
else
sampling.global_low_corner = db["space"]["global_low_corner"];
if ( db["sampling"].contains( "global_high_corner" ) )
sampling.global_high_corner =
db["sampling"]["global_high_corner"];
else
sampling.global_high_corner = db["space"]["global_high_corner"];

if ( sampling_type == "solidification_data" )
{
Expand All @@ -487,6 +513,14 @@ class Inputs
else
{
sampling.format = "default";
if ( sampling.fine_factor != 1 )
{
Info << "Warning: `default` sampling strategy not "
"currently allowed when performing interpolation; "
"`exaca` sampling will be used"
<< std::endl;
sampling.format = "exaca";
}
}

if ( db["sampling"].contains( "directory_name" ) )
Expand Down
90 changes: 62 additions & 28 deletions src/Finch_Run.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,13 +31,17 @@ class Layer
using memory_space = MemorySpace;
using sampling_type = Finch::SolidificationData<memory_space>;
sampling_type solidification_data_;
bool srdf_format;

Layer( Inputs& inputs, Grid<MemorySpace>& grid )
{
// Only construct if turned on - will otherwise default and immediately
// return from any member functions
if ( inputs.sampling.enabled )
solidification_data_ = sampling_type( inputs, grid );
{
srdf_format = ( inputs.sampling.fine_factor != 1 );
solidification_data_ = sampling_type( inputs, grid, srdf_format );
}
}

// Run the full timestepped loop
Expand Down Expand Up @@ -97,16 +101,43 @@ class Layer
auto owned_space = grid.getIndexSpace();
fd.solve( exec_space, owned_space, T, T0, beam_power, beam_pos );

// update boundaries
grid.updateBoundaries();

// communicate halos
grid.gather();
// update boundaries
grid.updateBoundaries();

solidification_data_.update( grid, time );
}

auto getSolidificationData() { return solidification_data_.get(); }
auto getSolidificationData( Grid<MemorySpace> grid, MPI_Comm comm,
Sampling sampling_inputs, bool write_data )
{
return solidification_data_.get( grid, comm, sampling_inputs,
write_data );
}

[[deprecated( "Use of getLowerSolidificationDataBounds() without a "
"communicator is deprecated." )]] std::array<double, 3>
getLowerSolidificationDataBounds()
{
return solidification_data_.getLowerBounds( MPI_COMM_WORLD );
}
[[deprecated( "Use of getUpperSolidificationDataBounds() without a "
"communicator is deprecated." )]] std::array<double, 3>
getUpperSolidificationDataBounds()
{
return solidification_data_.getUpperBounds( MPI_COMM_WORLD );
}

std::array<double, 3> getLowerSolidificationDataBounds( MPI_Comm comm )
{
return solidification_data_.getLowerBounds( comm );
}
std::array<double, 3> getUpperSolidificationDataBounds( MPI_Comm comm )
{
return solidification_data_.getUpperBounds( comm );
}

// Append next layer's solidification data to input_solidification_data
void appendSolidificationData(
Kokkos::View<double**, Kokkos::LayoutLeft, Kokkos::HostSpace>&
Expand Down Expand Up @@ -152,32 +183,35 @@ class Layer
events_prev_layers + events_this_layer;
}

auto writeSolidificationData( Sampling sampling_inputs, MPI_Comm comm )
{
return solidification_data_.write( sampling_inputs, comm );
}

[[deprecated( "Use of getLowerSolidificationDataBounds() without a "
"communicator is deprecated." )]] std::array<double, 3>
getLowerSolidificationDataBounds()
{
return solidification_data_.getLowerBounds( MPI_COMM_WORLD );
}
[[deprecated( "Use of getUpperSolidificationDataBounds() without a "
"communicator is deprecated." )]] std::array<double, 3>
getUpperSolidificationDataBounds()
auto writeSolidificationData( Grid<memory_space>& grid,
Sampling sampling_inputs, MPI_Comm comm )
{
return solidification_data_.getUpperBounds( MPI_COMM_WORLD );
return solidification_data_.write( sampling_inputs, grid, comm );
}

std::array<double, 3> getLowerSolidificationDataBounds( MPI_Comm comm )
{
return solidification_data_.getLowerBounds( comm );
}
std::array<double, 3> getUpperSolidificationDataBounds( MPI_Comm comm )
{
return solidification_data_.getUpperBounds( comm );
}
// [[deprecated( "Use of getLowerSolidificationDataBounds() without a "
// "communicator is deprecated." )]] std::array<double, 3>
// getLowerSolidificationDataBounds()
// {
// return solidification_data_.getLowerBounds( MPI_COMM_WORLD );
// }
// [[deprecated( "Use of getUpperSolidificationDataBounds() without a "
// "communicator is deprecated." )]] std::array<double, 3>
// getUpperSolidificationDataBounds()
// {
// return solidification_data_.getUpperBounds( MPI_COMM_WORLD );
// }
//
// std::array<double, 3> getLowerSolidificationDataBounds( MPI_Comm comm
// )
// {
// return solidification_data_.getLowerBounds( comm );
// }
// std::array<double, 3> getUpperSolidificationDataBounds( MPI_Comm comm
// )
// {
// return solidification_data_.getUpperBounds( comm );
// }
};

} // namespace Finch
Expand Down
Loading
Loading