IOSS: CGNS - workaround cgns fpp/parallel-io bug

Author: gdsjaar

packages/seacas/libraries/ioss/src/cgns/Iocgns_DatabaseIO.C

@@ -4,7 +4,7 @@
 // * Single Base.
 // * ZoneGridConnectivity is 1to1 with point lists for unstructured
 
-// Copyright(C) 1999-2024 National Technology & Engineering Solutions
+// Copyright(C) 1999-2025 National Technology & Engineering Solutions
 // of Sandia, LLC (NTESS).  Under the terms of Contract DE-NA0003525 with
 // NTESS, the U.S. Government retains certain rights in this software.
 //
@@ -79,6 +79,78 @@
 // extern char hdf5_access[64];
 
 namespace {
+  extern "C" {
+    // From private CGNS header: `cgio_internal_type.h`
+    typedef struct _cgns_io_ctx_t {
+      /* Flag indicating if HDF5 file accesses is PARALLEL or NATIVE */
+      char hdf5_access[64];
+#if CG_BUILD_PARALLEL
+      /* MPI-2 info object */
+      MPI_Comm pcg_mpi_comm;
+      int pcg_mpi_comm_size;
+      int pcg_mpi_comm_rank;
+      /* flag indicating if mpi_initialized was called */
+      int pcg_mpi_initialized;
+      MPI_Info pcg_mpi_info;
+      int64_t default_pio_mode;
+#endif
+    } cgns_io_ctx_t;
+    
+    extern cgns_io_ctx_t ctx_cgio; /* located in cgns_io.c */
+  }
+
+  // There is a bug in the CGNS library (4.4.0 and before) where it
+  // has a global symbol `ctx_cgio` which controls whether
+  // file-per-rank access is being used, or parallel io (single file,
+  // multiple ranks).  In an application (like many IOSS uses) that
+  // uses both access methods in the same execution, this can result
+  // in hangs and corruprtion due to the wrong access type being used
+  // at the wrong time (collective for a file-per-rank typically).
+  //
+  // The code below is a kluge to workaround this shortcoming in the
+  // CGNS library.  Before each file-per-rank access of the underlying
+  // file, the code below access the CGNS global `ctx_cgio` and sets
+  // it to non-parallel access.  After the access, the destructor of
+  // the class sets the global back to its previous value.
+  //
+  // The CGNS developers are aware of the issue and are looking at
+  // options.  See https://github.com/CGNS/CGNS/issues/835
+  struct ParallelGuard
+  {
+    ParallelGuard(IOSS_MAYBE_UNUSED bool yes_no)
+    {
+#if CG_BUILD_PARALLEL
+      m_wasSet = strcmp(ctx_cgio.hdf5_access, "PARALLEL") == 0;
+      if (m_wasSet != yes_no) {
+	m_changed = true;
+	if (yes_no) {
+	  strcpy(ctx_cgio.hdf5_access,"PARALLEL");
+	}
+	else {
+	  strcpy(ctx_cgio.hdf5_access,"NATIVE");
+	}
+      }
+#endif
+    }
+    ~ParallelGuard() 
+    {
+#if CG_BUILD_PARALLEL
+      if (m_changed) {
+	if (m_wasSet) {
+	  strcpy(ctx_cgio.hdf5_access,"PARALLEL");
+	}
+	else {
+	  strcpy(ctx_cgio.hdf5_access,"NATIVE");
+	}
+      }
+#endif
+    }
+#if CG_BUILD_PARALLEL
+    bool m_wasSet{false};
+    bool m_changed{false};
+#endif
+  };
+
   size_t global_to_zone_local_idx(size_t i, const Ioss::Map *block_map, const Ioss::Map &nodeMap,
                                   bool isParallel)
   {
@@ -996,7 +1068,7 @@ namespace Iocgns {
       Ioss::IJK_t global_ijk;
       Ioss::IJK_t offset_ijk;
 
-      zone_data[id++]; // proc field. Not currently used.
+      id++; // proc field. Not currently used.
       unpack(id, Data(zone_data), local_ijk.data(), 3);
       unpack(id, Data(zone_data), global_ijk.data(), 3);
       unpack(id, Data(zone_data), offset_ijk.data(), 3);
@@ -1533,6 +1605,7 @@ namespace Iocgns {
           "ERROR: CGNS: Too many bases; only support files with a single bases at this time");
     }
 
+    ParallelGuard serial(0);
     get_step_times_nl();
 
     if (open_create_behavior() == Ioss::DB_APPEND) {
@@ -1862,6 +1935,7 @@ namespace Iocgns {
     };
     // End of lambda...
 
+    ParallelGuard serial(0);
     if (role == Ioss::Field::MESH) {
       if (field.get_name() == "mesh_model_coordinates_x") {
         // Use the lambda...
@@ -2000,6 +2074,7 @@ namespace Iocgns {
       return 0;
     }
 
+    ParallelGuard serial(0);
     Ioss::Field::RoleType role = field.get_role();
     if (role == Ioss::Field::TRANSIENT) {
       // Get the StructuredBlock that this NodeBlock is contained in:
@@ -2069,6 +2144,7 @@ namespace Iocgns {
   int64_t DatabaseIO::get_field_internal(const Ioss::ElementBlock *eb, const Ioss::Field &field,
                                          void *data, size_t data_size) const
   {
+    ParallelGuard serial(0);
     size_t num_to_get = field.verify(data_size);
     if (num_to_get > 0) {
 
@@ -2210,6 +2286,7 @@ namespace Iocgns {
     int                   zone = Iocgns::Utils::get_db_zone(sb);
 
     cgsize_t num_to_get = field.verify(data_size);
+    ParallelGuard serial(0);
 
     // In this routine, if isParallel, then reading file-per-processor; not parallel io from single
     // file.
@@ -2398,6 +2475,7 @@ namespace Iocgns {
   int64_t DatabaseIO::get_field_internal(const Ioss::SideBlock *sb, const Ioss::Field &field,
                                          void *data, size_t data_size) const
   {
+    ParallelGuard serial(0);
     int base = sb->get_property("base").get_int();
     int zone = Iocgns::Utils::get_db_zone(sb);
     int sect = sb->get_property("section").get_int();
@@ -2554,6 +2632,7 @@ namespace Iocgns {
 
     // In this routine, if isParallel, then writing file-per-processor; not parallel io to single
     // file.
+    ParallelGuard serial(0);
     if (isParallel && num_to_get == 0) {
       return 0;
     }
@@ -2664,6 +2743,7 @@ namespace Iocgns {
   int64_t DatabaseIO::put_field_internal(const Ioss::ElementBlock *eb, const Ioss::Field &field,
                                          void *data, size_t data_size) const
   {
+    ParallelGuard serial(0);
     size_t num_to_get = field.verify(data_size);
     if (num_to_get > 0) {
 
@@ -2835,6 +2915,7 @@ namespace Iocgns {
       return put_field_internal_sub_nb(nb, field, data, data_size);
     }
 
+    ParallelGuard serial(0);
     // Instead of outputting a global nodeblock's worth of data,
     // the data is output a "zone" at a time.
     // The m_globalToBlockLocalNodeMap[zone] map is used (Ioss::Map pointer)
@@ -3007,6 +3088,7 @@ namespace Iocgns {
     int                         zone       = Iocgns::Utils::get_db_zone(sb);
     cgsize_t                    num_to_get = field.verify(data_size);
 
+    ParallelGuard serial(0);
     // In this routine, if isParallel, then writing file-per-processor; not parallel io to single
     // file.
     if (isParallel && num_to_get == 0) {
@@ -3070,6 +3152,7 @@ namespace Iocgns {
   int64_t DatabaseIO::put_field_internal(const Ioss::SideBlock *sb, const Ioss::Field &field,
                                          void *data, size_t data_size) const
   {
+    ParallelGuard serial(0);
     const Ioss::EntityBlock *parent_block = sb->parent_block();
     if (parent_block == nullptr) {
       IOSS_ERROR(fmt::format(

packages/seacas/libraries/ioss/src/cgns/Iocgns_Utils.C

@@ -1266,6 +1266,9 @@ size_t Iocgns::Utils::common_write_metadata(int file_ptr, const Ioss::Region &re
       for (size_t i = 0; i < 3; i++) {
         bc_range[idx++] = bc.m_rangeEnd[i];
       }
+      if (is_parallel_io) {
+	region.get_database()->progress(fmt::format("\t\tBC Range calculation, {}.", bc.m_bcName));
+      }
     }
 
     if (is_parallel_io) {