#56: implement parallel meshes as smaller instances of a global mesh

cppfd/CMakeLists.txt

@@ -9,6 +9,7 @@ include_directories(${CMAKE_CURRENT_BINARY_DIR})
 
 # Setting options
 option(OUTPUT_VTK_FILES "When OFF allows for a non-VTK build" OFF)
+option(USE_MPI "When OFF allows for a non-MPI build" OFF)
 
 # Find required dependencies
 find_package(Kokkos REQUIRED)
@@ -18,6 +19,10 @@ if(OUTPUT_VTK_FILES)
   find_package(VTK REQUIRED COMPONENTS vtkCommonCore vtkIOXML)
 endif()
 
+if(USE_MPI)
+  find_package(MPI REQUIRED)
+endif()
+
 find_package(GTest CONFIG REQUIRED)
 
 # Try to find ccache to speed up compilation
@@ -40,10 +45,22 @@ add_executable(allTests allTests.cpp)
 message(STATUS "VTK libraries: ${VTK_LIBRARIES}")
 
 # Link targets to libraries
-target_link_libraries(mesh_chunk Kokkos::kokkos ${VTK_LIBRARIES})
-target_link_libraries(parallel_mesh Kokkos::kokkos ${VTK_LIBRARIES})
+if (${USE_MPI} AND MPI_FOUND)
+    MESSAGE(STATUS "MPI found. Enabling MPI.")
+    target_link_libraries(mesh_chunk MPI::MPI_CXX)
+    target_link_libraries(parallel_mesh MPI::MPI_CXX)
+    target_link_libraries(boundary_conditions MPI::MPI_CXX)
+    target_link_libraries(solver MPI::MPI_CXX)
+    target_link_libraries(test MPI::MPI_CXX)
+    target_link_libraries(parallel_test MPI::MPI_CXX)
+    target_link_libraries(allTests MPI::MPI_CXX)
+elseif(${USE_MPI} AND NOT MPI_FOUND)
+    MESSAGE(WARNING "MPI not found. Disabling MPI.")
+endif()
+target_link_libraries(mesh_chunk boundary_conditions Kokkos::kokkos ${VTK_LIBRARIES})
+target_link_libraries(parallel_mesh mesh_chunk Kokkos::kokkos ${VTK_LIBRARIES})
 target_link_libraries(boundary_conditions Kokkos::kokkos ${VTK_LIBRARIES})
-target_link_libraries(solver Kokkos::kokkos Kokkos::kokkoskernels  ${VTK_LIBRARIES})
-target_link_libraries(test Kokkos::kokkos Kokkos::kokkoskernels ${VTK_LIBRARIES})
-target_link_libraries(parallel_test Kokkos::kokkos Kokkos::kokkoskernels ${VTK_LIBRARIES})
-target_link_libraries(allTests Kokkos::kokkos Kokkos::kokkoskernels ${VTK_LIBRARIES} GTest::gtest)
+target_link_libraries(solver parallel_mesh Kokkos::kokkos Kokkos::kokkoskernels ${VTK_LIBRARIES})
+target_link_libraries(test solver mesh_chunk Kokkos::kokkos Kokkos::kokkoskernels ${VTK_LIBRARIES})
+target_link_libraries(parallel_test solver Kokkos::kokkos Kokkos::kokkoskernels ${VTK_LIBRARIES})
+target_link_libraries(allTests solver parallel_mesh Kokkos::kokkos Kokkos::kokkoskernels ${VTK_LIBRARIES} GTest::gtest)

cppfd/allTests.cpp

@@ -4,9 +4,9 @@
 #include <Kokkos_Core.hpp>
 #include <gtest/gtest.h>
 
-#include "mesh_chunk.cpp"
-#include "boundary_conditions.cpp"
-#include "solver.cpp"
+#include "mesh_chunk.h"
+#include "boundary_conditions.h"
+#include "solver.h"
 
 struct solver_test : testing::Test{
   std::unique_ptr<Solver> solver;
@@ -20,6 +20,10 @@ struct solver_test : testing::Test{
     double t_final = 0.1;
     double max_C = 0.5;
     uint64_t n_cells = 3;
+    uint64_t n_parallel_meshes_x = 1;
+    uint64_t n_parallel_meshes_y = 1;
+    uint64_t n_colors_x = 1;
+    uint64_t n_colors_y = 1;
 
     // create mesh
     std::map<PointIndexEnum, PointTypeEnum> point_types = {
@@ -50,7 +54,7 @@ struct solver_test : testing::Test{
 
     // instantiate solver
     this->solver = std::make_unique<Solver>
-      (mesh, b_c, delta_t, t_final, density, dynamic_viscosity, max_C, 0);
+      (mesh, b_c, delta_t, t_final, density, dynamic_viscosity, max_C, 0, 25, 25, 1, n_parallel_meshes_x, n_parallel_meshes_y, n_colors_x, n_colors_y);
   }
 };
 

cppfd/cfd_config.h.in

@@ -1,2 +1,3 @@
 #pragma once
 #cmakedefine OUTPUT_VTK_FILES
+#cmakedefine USE_MPI

cppfd/mesh_chunk.cpp

@@ -146,6 +146,29 @@ get_pressure(uint64_t i, uint64_t j) const{
   else
     return this->pressure(k);
 }
+
+#ifdef USE_MPI
+void MeshChunk::
+mpi_send_border_velocity_x(double border_velocity_x, uint64_t pos_x, uint64_t pos_y, uint64_t dest_rank, uint8_t border) {
+	MPI_Send(&border_velocity_x, 1, MPI_DOUBLE, dest_rank, border, MPI_COMM_WORLD);
+}
+
+void MeshChunk::
+mpi_send_border_velocity_y(double border_velocity_y, uint64_t pos_x, uint64_t pos_y, uint64_t dest_rank, uint8_t border) {
+	MPI_Send(&border_velocity_y, 1, MPI_DOUBLE, dest_rank, border, MPI_COMM_WORLD);
+}
+
+void MeshChunk::
+mpi_receive_border_velocity_x(double border_velocity_x, uint64_t pos_x, uint64_t pos_y, uint64_t dest_rank, uint8_t border, MPI_Status status) {
+	MPI_Recv(&border_velocity_x, 1, MPI_DOUBLE, dest_rank, border, MPI_COMM_WORLD, &status);
+}
+
+void MeshChunk::
+mpi_receive_border_velocity_y(double border_velocity_y, uint64_t pos_x, uint64_t pos_y, uint64_t dest_rank, uint8_t border, MPI_Status status) {
+	MPI_Recv(&border_velocity_y, 1, MPI_DOUBLE, dest_rank, border, MPI_COMM_WORLD, &status);
+}
+#endif
+
 #ifdef OUTPUT_VTK_FILES
 vtkSmartPointer<vtkUniformGrid> MeshChunk::
 make_VTK_uniform_grid() const{
@@ -211,4 +234,4 @@ write_vti(const std::string& file_name) const{
   // return fill name with extension
   return full_file_name;
 }
-#endif
+#endif

cppfd/mesh_chunk.h

@@ -7,20 +7,24 @@
 
 #include <Kokkos_Core.hpp>
 
+#ifdef USE_MPI
+#include <mpi.h>
+#endif
+
 #ifdef OUTPUT_VTK_FILES
 #include <vtkSmartPointer.h>
 #endif
 
 enum struct PointIndexEnum : int8_t {
-  CORNER_0 = 0,
-  CORNER_1 = 1,
-  CORNER_2 = 2,
-  CORNER_3 = 3,
-  EDGE_0 = 4,
-  EDGE_1 = 5,
-  EDGE_2 = 6,
-  EDGE_3 = 7,
-  INTERIOR = 8
+  CORNER_0 = 0, // Bottom Left
+  CORNER_1 = 1, // Bottom Right
+  CORNER_2 = 2, // Top Right
+  CORNER_3 = 3, // Top Left
+  EDGE_0 = 4, // Bottom
+  EDGE_1 = 5, // Right
+  EDGE_2 = 6, // Top
+  EDGE_3 = 7, // Left
+  INTERIOR = 8 // Interior
 };
 
 enum struct PointTypeEnum : int8_t {
@@ -31,6 +35,13 @@ enum struct PointTypeEnum : int8_t {
   INVALID = 4
 };
 
+enum struct Border : int8_t {
+  BOTTOM = 0,
+  LEFT = 1,
+  RIGHT = 2,
+  TOP = 3
+};
+
 #ifdef OUTPUT_VTK_FILES
 class vtkUniformGrid;
 #endif
@@ -70,6 +81,16 @@ class MeshChunk
     // setter to assign new mesh cell data
     void set_pressure(Kokkos::View<double*> p) { this->pressure = p; }
 
+    #ifdef USE_MPI
+    // mpi sends to other mesh chunk
+    void mpi_send_border_velocity_x(double border_velocity_x, uint64_t pos_x, uint64_t pos_y, uint64_t dest_rank, uint8_t border);
+    void mpi_send_border_velocity_y(double border_velocity_y, uint64_t pos_x, uint64_t pos_y, uint64_t dest_rank, uint8_t border);
+
+    // mpi receive from other mesh chunk
+    void mpi_receive_border_velocity_x(double border_velocity_x, uint64_t pos_x, uint64_t pos_y, uint64_t source_rank, uint8_t border, MPI_Status status);
+    void mpi_receive_border_velocity_y(double border_velocity_y, uint64_t pos_x, uint64_t pos_y, uint64_t source_rank, uint8_t border, MPI_Status status);
+    #endif
+
     // converter to VTK uniform grid
     #ifdef OUTPUT_VTK_FILES
     vtkSmartPointer<vtkUniformGrid> make_VTK_uniform_grid() const;