Add parallel domain mismatch checking

Author: siramok

src/libs/rover/domain.cpp

@@ -91,6 +91,12 @@ Domain::get_num_energy_groups()
   return m_engine->get_num_energy_groups();
 }
 
+bool
+Domain::get_field_mismatch_error()
+{
+  return m_engine->get_field_mismatch_error();
+}
+
 void
 Domain::set_dataset(vtkmDataSet &dataset)
 {

src/libs/rover/domain.hpp

@@ -40,6 +40,7 @@ class Domain
   vtkmRange get_primary_range();
   void set_global_bounds(vtkm::Bounds bounds);
   const int get_num_energy_groups();
+  bool get_field_mismatch_error();
 protected:
   std::shared_ptr<Engine> m_engine;
   vtkmDataSet             m_dataset;

src/libs/rover/engine.cpp

@@ -17,6 +17,7 @@ namespace rover
 Engine::Engine()
 {
   m_tracer = nullptr;
+  m_field_mismatch_error = false;
 }
 
 Engine::~Engine()
@@ -134,15 +135,19 @@ Engine::get_num_energy_groups()
 
     if (num_absorption_bins != num_emission_bins)
     {
-      ASCENT_LOG_ERROR("Error - Engine::get_num_energy_groups: number of energy groups in absorption field ("
-                       << num_absorption_bins << ") does not match number of bins in emission field ("
-                       << num_emission_bins << ")");
+      m_field_mismatch_error = true;
     }
   }
 
   return static_cast<int>(num_absorption_bins);
 }
 
+bool
+Engine::get_field_mismatch_error()
+{
+  return m_field_mismatch_error;
+}
+
 vtkmRange
 Engine::get_primary_range()
 {

src/libs/rover/engine.hpp

@@ -44,13 +44,15 @@ class Engine
   void partial_trace(Ray32 &rays, PartialVector32 &partials);
   void partial_trace(Ray64 &rays, PartialVector64 &partials);
   int  get_num_energy_groups();
+  bool get_field_mismatch_error();
   vtkmRange get_primary_range();
   void set_primary_range(const vtkmRange &range);
   void set_composite_background(bool on);
 
 protected:
   vtkmDataSet m_dataset;
   vtkh::rendering::ConnectivityProxy *m_tracer;
+  bool m_field_mismatch_error;
 
   template<typename Precision>
   void init_emission(vtkmRayTracing::Ray<Precision> &rays,

src/libs/rover/typed_scheduler.cpp

@@ -72,9 +72,16 @@ int
 TypedScheduler<FloatType>::get_global_num_energy_groups()
 {
   int num_energy_groups = 1;
+  int has_field_mismatch = 0;
+
   for (auto& domain : m_domains)
   {
     num_energy_groups = std::max(num_energy_groups, domain.get_num_energy_groups());
+    // Check if this domain had a field mismatch
+    if (domain.get_field_mismatch_error())
+    {
+      has_field_mismatch = 1;
+    }
   }
 
 #ifdef ROVER_PARALLEL
@@ -92,8 +99,23 @@ TypedScheduler<FloatType>::get_global_num_energy_groups()
                 "Local: " << num_energy_groups << ", Global max: " << mpi_max_energy_groups);
   }
 
+  // Check that all ranks agree on field mismatch state
+  int global_field_mismatch = 0;
+  MPI_Allreduce(&has_field_mismatch, &global_field_mismatch, 1, MPI_INT, MPI_MAX, m_comm_handle);
+  if (global_field_mismatch)
+  {
+    ASCENT_LOG_ERROR("Error - TypedScheduler::get_global_num_energy_groups: "
+                     "mismatched nunmber of absorption and emission fields detected on one or more ranks");
+  }
+
   time = timer.GetElapsedTime();
   ROVER_DATA_ADD("get_global_num_energy_groups_all_reduce", time);
+#else
+  if (has_field_mismatch)
+  {
+    ASCENT_LOG_ERROR("Error - TypedScheduler::get_global_num_energy_groups: "
+                     "mismatched number of absorption and emission fields");
+  }
 #endif
 
   ROVER_INFO("Global number of energy groups" << num_energy_groups);