#3: comm: first step toward parameterized testing

Author: nlslatt

src/vt-lb/comm/MPI/comm_mpi.h

@@ -170,12 +170,7 @@ struct CommMPI {
   template <typename T>
   using HandleType = ClassHandle<T>;
 
-  /**
-   * \brief Construct a new CommMPI instance
-   * \param comm The MPI communicator to use (defaults to MPI_COMM_WORLD)
-   */
-  explicit CommMPI(MPI_Comm comm = MPI_COMM_WORLD) : comm_(comm) { }
-
+  CommMPI() = default;
   CommMPI(CommMPI const&) = delete;
   CommMPI(CommMPI&&) = delete;
   CommMPI& operator=(CommMPI const&) = delete;
@@ -194,8 +189,14 @@ struct CommMPI {
    * \param argc Pointer to argument count
    * \param argv Pointer to argument array
    */
-  void init(int& argc, char**& argv) {
-    MPI_Init(&argc, &argv);
+  void init(int& argc, char**& argv, MPI_Comm comm = MPI_COMM_NULL) {
+    if (comm == MPI_COMM_NULL) {
+      MPI_Init(&argc, &argv);
+      comm_ = MPI_COMM_WORLD;
+    } else {
+      interop_mode_ = true;
+      comm_ = comm;
+    }
     MPI_Comm_rank(comm_, &cached_rank_);
     MPI_Comm_size(comm_, &cached_size_);
     initTermination();
@@ -216,8 +217,10 @@ struct CommMPI {
    * \brief Finalize MPI
    */
   void finalize() {
-    // printf("%d: Finalizing MPI\n", cached_rank_);
-    MPI_Finalize();
+    if (!interop_mode_) {
+      // printf("%d: Finalizing MPI\n", cached_rank_);
+      MPI_Finalize();
+    }
     comm_ = MPI_COMM_NULL;
   }
 
@@ -444,6 +447,7 @@ struct CommMPI {
 private:
   void initTermination();
 
+  bool interop_mode_ = false;
   MPI_Comm comm_ = MPI_COMM_NULL;
   std::list<std::tuple<MPI_Request, std::unique_ptr<char[]>>> pending_;
   int next_class_index_ = 0;

src/vt-lb/comm/vt/comm_vt.cc

@@ -47,8 +47,13 @@
 
 namespace vt_lb::comm {
 
-void CommVT::init(int& argc, char**& argv) {
-  vt::initialize(argc, argv);
+void CommVT::init(int& argc, char**& argv, MPI_Comm comm) {
+  if (comm == MPI_COMM_NULL) {
+    vt::initialize(argc, argv);
+  } else {
+    // interop mode
+    vt::initialize(argc, argv, &comm);
+  }
   vt::theTerm()->addDefaultAction([this]{ terminated_ = true; });
 }
 

src/vt-lb/comm/vt/comm_vt.h

@@ -65,7 +65,7 @@ struct CommVT {
   CommVT(vt::EpochType epoch);
 
 public:
-  void init(int& argc, char**& argv);
+  void init(int& argc, char**& argv, MPI_Comm comm = MPI_COMM_NULL);
   void finalize();
   int numRanks() const;
   int getRank() const;

tests/unit/test_helpers.h

@@ -44,7 +44,6 @@
 #if !defined INCLUDED_VT_LB_UNIT_TEST_HELPERS_H
 #define INCLUDED_VT_LB_UNIT_TEST_HELPERS_H
 
-#include "vt/context/context.h"
 #include <mpi.h>
 #include <gtest/gtest.h>
 #include <sstream>
@@ -58,23 +57,23 @@ extern char** test_argv;
  * Maximum number of ranks/nodes detected by CMake on this machine.
  * Defaults to number of processors detected on the host system.
  */
-constexpr vt::NodeType CMAKE_DETECTED_MAX_NUM_NODES = vt_detected_max_num_nodes;
+//constexpr vt::NodeType CMAKE_DETECTED_MAX_NUM_NODES = vt_detected_max_num_nodes;
 
 /**
  * Check whether we're oversubscribing on the current execution.
  * This is using MPI because it can be used before vt initializes.
  */
-inline bool isOversubscribed() {
-  // be sure to only call this from parallel tests
-  int init = 0;
-  MPI_Initialized(&init);
-  if (!init) {
-    MPI_Init(&test_argc, &test_argv);
-  }
-  int num_ranks = 0;
-  MPI_Comm_size(MPI_COMM_WORLD, &num_ranks);
-  return num_ranks > CMAKE_DETECTED_MAX_NUM_NODES;
-}
+//inline bool isOversubscribed() {
+//  // be sure to only call this from parallel tests
+//  int init = 0;
+//  MPI_Initialized(&init);
+//  if (!init) {
+//    MPI_Init(&test_argc, &test_argv);
+//  }
+//  int num_ranks = 0;
+//  MPI_Comm_size(MPI_COMM_WORLD, &num_ranks);
+//  return num_ranks > CMAKE_DETECTED_MAX_NUM_NODES;
+//}
 
 /**
  * Get a unique filename based on the unit test name.
@@ -94,12 +93,12 @@ inline std::string getUniqueFilename(const std::string& ext = "") {
  * concurrently-running tests will not cause file system race conditions.
  * Do not call this from .nompi.cc tests or from addAdditionalArgs().
  */
-inline std::string getUniqueFilenameWithRanks(const std::string& ext = "") {
-  auto ranks = vt::theContext()->getNumNodes();
-  std::stringstream ss;
-  ss << getUniqueFilename() << "_" << ranks << ext;
-  return ss.str();
-}
+//inline std::string getUniqueFilenameWithRanks(const std::string& ext = "") {
+//  auto ranks = vt::theContext()->getNumNodes();
+//  std::stringstream ss;
+//  ss << getUniqueFilename() << "_" << ranks << ext;
+//  return ss.str();
+//}
 
 /**
  * The following helper macros (these have to be macros, because GTEST_SKIP
@@ -116,7 +115,7 @@ inline std::string getUniqueFilenameWithRanks(const std::string& ext = "") {
  */
 #define SET_MAX_NUM_NODES_CONSTRAINT(max_req_num_nodes)                   \
 {                                                                         \
-  auto const num_nodes = vt::theContext()->getNumNodes();                     \
+  auto const num_nodes = comm.numRanks();                                 \
   if (num_nodes > max_req_num_nodes) {                                    \
     GTEST_SKIP() << fmt::format(                                          \
       "Skipping the run on {} nodes. This test should run on at most {} " \
@@ -132,7 +131,7 @@ inline std::string getUniqueFilenameWithRanks(const std::string& ext = "") {
  */
 #define SET_MIN_NUM_NODES_CONSTRAINT(min_req_num_nodes)                   \
 {                                                                         \
-  auto const num_nodes = vt::theContext()->getNumNodes();                     \
+  auto const num_nodes = comm.numRanks();                                 \
   if (num_nodes < min_req_num_nodes) {                                    \
     GTEST_SKIP() << fmt::format(                                          \
       "Skipping the run on {} nodes. This test should run on at least {} "\
@@ -148,7 +147,7 @@ inline std::string getUniqueFilenameWithRanks(const std::string& ext = "") {
  */
 #define SET_NUM_NODES_CONSTRAINT(req_num_nodes)                           \
 {                                                                         \
-  auto const num_nodes = vt::theContext()->getNumNodes();                     \
+  auto const num_nodes = comm.numRanks();                                 \
   if (num_nodes != req_num_nodes) {                                       \
     GTEST_SKIP() << fmt::format(                                          \
       "Skipping the run on {} nodes. This test should run only on {} "    \

tests/unit/test_parallel_harness.h

@@ -50,13 +50,17 @@
 #include "test_config.h"
 #include "test_harness.h"
 
-#include "vt/transport.h"
+#include <vt-lb/comm/vt/comm_vt.h>
+#include <vt-lb/comm/MPI/comm_mpi.h>
 
 namespace vt_lb { namespace tests { namespace unit {
 
 extern int test_argc;
 extern char** test_argv;
 
+using CommType = vt_lb::comm::CommVT;
+//using CommType = vt_lb::comm::CommMPI;
+
 template <typename TestBase>
 struct TestParallelHarnessAny : TestHarnessAny<TestBase> {
   virtual void SetUp() override {
@@ -68,41 +72,44 @@ struct TestParallelHarnessAny : TestHarnessAny<TestBase> {
     if (!init) {
       MPI_Init(&test_argc, &test_argv);
     }
-    MPI_Comm comm = MPI_COMM_WORLD;
+    MPI_Comm mpi_comm = MPI_COMM_WORLD;
     auto const new_args = injectAdditionalArgs(test_argc, test_argv);
     auto custom_argc = new_args.first;
     auto custom_argv = new_args.second;
     vtAssert(
       custom_argv[custom_argc] == nullptr,
       "The value of argv[argc] should always be 0"
     );
-    // communicator is duplicated.
-    vt::initialize(custom_argc, custom_argv, &comm, nullptr, true);
+    comm.init(custom_argc, custom_argv, mpi_comm);
 
 #if DEBUG_TEST_HARNESS_PRINT
-    auto const& my_node = vt::theContext()->getNode();
-    auto const& num_nodes = vt::theContext()->getNumNodes();
-    fmt::print("my_node={}, num_nodes={}\n", my_node, num_nodes);
+    auto const& my_rank = comm.getRank();
+    auto const& num_ranks = comm.numRanks();
+    fmt::print("my_rank={}, num_ranks={}\n", my_rank, num_ranks);
 #endif
   }
 
   virtual void TearDown() override {
     try {
-      vt::theSched()->runSchedulerWhile([] { return !vt::rt->isTerminated(); });
+      while (comm.poll()) {
+      }
     } catch (std::exception& e) {
       ADD_FAILURE() << fmt::format("Caught an exception: {}\n", e.what());
     }
 
 #if DEBUG_TEST_HARNESS_PRINT
-    auto const& my_node = vt::theContext()->getNode();
-    fmt::print("my_node={}, tearing down runtime\n", my_node);
+    auto const& my_rank = comm.getRank();
+    fmt::print("my_rank={}, tearing down runtime\n", my_rank);
 #endif
 
-    vt::finalize();
+    comm.finalize();
 
     TestHarnessAny<TestBase>::TearDown();
   }
 
+public:
+  CommType comm;
+
 protected:
   template <typename Arg>
   void addArgs(Arg& arg) {