Fix HDF5 dataset creation for large non-scalar variables

CMakeLists.txt

@@ -66,9 +66,17 @@ option (HDF5_USE_LOSSY_COMPRESSION  "Enable lossy compression methods in HDF5 (e
 option (PIO_ENABLE_TESTS  "Enable the testing builds"                       OFF)
 option (PIO_ENABLE_LARGE_TESTS  "Enable large (file, processes) tests"      OFF)
 option (PIO_VALGRIND_CHECK  "Enable memory leak check using valgrind"       OFF)
-#===== Dependent Options =====
-include(CMakeDependentOption)
-cmake_dependent_option (PIO_TEST_CLOSE_OPEN_FOR_SYNC  "SCORPIO fortran tests will close+open for sync" ON "WITH_ADIOS2" OFF)
+option (PIO_TEST_CLOSE_OPEN_FOR_SYNC "Enable close+open to sync file output in tests" OFF)
+
+if (WITH_ADIOS2)
+  message(STATUS "Enabling close+open to sync file output for SCORPIO tests (default since ADIOS is enabled)")
+  set(PIO_TEST_CLOSE_OPEN_FOR_SYNC ON CACHE BOOL "Enable close+open to sync file output in tests" FORCE)
+endif ()
+
+if (WITH_HDF5 AND (NOT PIO_TEST_CLOSE_OPEN_FOR_SYNC))
+  message(STATUS "Enabling close+open to sync file output for SCORPIO tests (default since HDF5 is enabled)")
+  set(PIO_TEST_CLOSE_OPEN_FOR_SYNC ON CACHE BOOL "Enable close+open to sync file output in tests" FORCE)
+endif ()
 
 # Get the name of the build host
 cmake_host_system_information (RESULT FQDN_SITENAME QUERY FQDN)
@@ -229,6 +237,20 @@ else()
   message(STATUS "Reserving some extra space in the header when creating NetCDF files, requested bytes = " ${PIO_RESERVED_FILE_HEADER_SIZE} " (default)")
 endif()
 
+# Set chunk size (in bytes) for HDF5/PnetCDF chunked variables. Default is 4 MB, same as NetCDF4.
+set(DEF_SPIO_CHUNK_SIZE 4194304)
+if(DEFINED PIO_CHUNK_SIZE)
+  if(PIO_CHUNK_SIZE GREATER_EQUAL 1)
+    message(STATUS "Setting chunk size (in bytes) for HDF5/PnetCDF chunked variables, requested bytes = " ${PIO_CHUNK_SIZE})
+  else()
+    message(WARNING "User-defined PIO_CHUNK_SIZE is invalid, setting it to " ${DEF_SPIO_CHUNK_SIZE} " (default)")
+    set(PIO_CHUNK_SIZE ${DEF_SPIO_CHUNK_SIZE})
+  endif()
+else()
+  set(PIO_CHUNK_SIZE ${DEF_SPIO_CHUNK_SIZE})
+  message(STATUS "Setting chunk size (in bytes) for HDF5/PnetCDF chunked variables, requested bytes = " ${PIO_CHUNK_SIZE} " (default)")
+endif()
+
 if(WITH_ADIOS2)
   # Maximum number of I/O decompositions registered with ADIOS type
   set(DEF_SPIO_MAX_ADIOS_DECOMPS 65536)

examples/c/test_hdf5.c

@@ -43,6 +43,12 @@ int main(int argc, char* argv[])
     int dimid_ncol;
     int dimid_sample;
     int dimid_ship;
+    int dimid_nCells;
+    int dimid_nEdges;
+    int dimid_nVertices;
+    int dimid_nVertLevels;
+    int dimid_nVertLevelsP1;
+    int dimid_StrLen;
 
     int varid;
     int varid_time;
@@ -57,6 +63,7 @@ int main(int argc, char* argv[])
     int varid_U;
     int varid_sample;
     int varid_ship;
+    int varid_normalVelocity;
 
     int att_id;
     int att_type;
@@ -594,6 +601,86 @@ int main(int argc, char* argv[])
 
         ret = PIOc_closefile(ncid); ERR
 
+        if (my_rank == 0)
+        {
+            printf("Test reading %s with NETCDF4 IO type end\n", filename);
+            fflush(stdout);
+        }
+#endif
+
+        /* Simulate a history file of E3SM G case with large dimension lengths (compset GMPAS-NYF, res T62_oRRS18to6v3) */
+        snprintf(filename, PIO_MAX_NAME, "test_hdf5_g_case_rearr_%d.nc", rearranger[r]);
+
+        if (my_rank == 0)
+        {
+            printf("Test writing %s with HDF5 IO type start\n", filename);
+            fflush(stdout);
+        }
+
+        ncid = -1;
+        ret = PIOc_createfile(iosysid, &ncid, &format, filename, PIO_CLOBBER); ERR
+
+        ret = PIOc_def_dim(ncid, "Time", PIO_UNLIMITED, &dimid_time); ERR
+        ret = PIOc_def_dim(ncid, "nCells", 3693225, &dimid_nCells); ERR
+        ret = PIOc_def_dim(ncid, "nEdges", 11135652, &dimid_nEdges); ERR
+        ret = PIOc_def_dim(ncid, "nVertices", 7441216, &dimid_nVertices); ERR
+        ret = PIOc_def_dim(ncid, "nVertLevels", 81, &dimid_nVertLevels); ERR
+        ret = PIOc_def_dim(ncid, "nVertLevelsP1", 81, &dimid_nVertLevelsP1); ERR
+        ret = PIOc_def_dim(ncid, "StrLen", 64, &dimid_StrLen); ERR
+
+        dimids[0] = dimid_time;
+        dimids[1] = dimid_nEdges;
+        dimids[2] = dimid_nVertLevels;
+        ret = PIOc_def_var(ncid, "normalVelocity", PIO_DOUBLE, 3, dimids, &varid_normalVelocity); ERR
+
+        ret = PIOc_enddef(ncid); ERR
+
+        ret = PIOc_closefile(ncid); ERR
+
+        if (my_rank == 0)
+        {
+            printf("Test writing %s with HDF5 IO type end\n", filename);
+            fflush(stdout);
+        }
+
+#ifdef _NETCDF4
+        /* Direct read support for HDF5 IO type is not implemented yet: SCORPIO implicitly switches HDF5 type to
+         * NETCDF4 type for reading back output files (generated by HDF5 type, fully NETCDF4 compatible). */
+        if (my_rank == 0)
+        {
+            printf("Test reading %s with NETCDF4 IO type start\n", filename);
+            fflush(stdout);
+        }
+
+        ncid = -1;
+        ret = PIOc_openfile(iosysid, &ncid, &format, filename, PIO_NOWRITE); ERR
+
+        ndims = -1;
+        ret = PIOc_inq_ndims(ncid, &ndims); ERR
+        assert(ndims == 7);
+
+        dimid_time = -1;
+        ret = PIOc_inq_dimid(ncid, "Time", &dimid_time); ERR
+        assert(dimid_time >= 0 && dimid_time < ndims);
+
+        nvars = -1;
+        ret = PIOc_inq_nvars(ncid, &nvars); ERR
+        assert(nvars == 1);
+
+        PIO_Offset dimlen_nCells = -1;
+        ret = PIOc_inq_dimlen(ncid, dimid_nCells, &dimlen_nCells); ERR
+        assert(dimlen_nCells == 3693225);
+
+        PIO_Offset dimlen_nEdges = -1;
+        ret = PIOc_inq_dimlen(ncid, dimid_nEdges, &dimlen_nEdges); ERR
+        assert(dimlen_nEdges == 11135652);
+
+        PIO_Offset dimlen_nVertices = -1;
+        ret = PIOc_inq_dimlen(ncid, dimid_nVertices, &dimlen_nVertices); ERR
+        assert(dimlen_nVertices == 7441216);
+
+        ret = PIOc_closefile(ncid); ERR
+
         if (my_rank == 0)
         {
             printf("Test reading %s with NETCDF4 IO type end\n", filename);

examples/cxx/CMakeLists.txt

@@ -48,8 +48,16 @@ LINK_DIRECTORIES(${PIO_LIB_DIR})
 #==============================================================================
 #  BUILD EXECUTABLE
 #==============================================================================
+SET(SRC ${SCORPIO_SOURCE_DIR}/util/argparser.cxx)
+add_library(argparser OBJECT ${SRC})
+
 SET(SRC examplePio.cxx)
 add_spio_executable(examplePio_cxx TRUE "" ${SRC})
+target_link_libraries(examplePio_cxx PRIVATE $<TARGET_OBJECTS:argparser>)
+
+SET(SRC e3sm_fgi.cpp e3sm_f.cpp e3sm_g.cpp e3sm_i.cpp)
+add_spio_executable(e3sm_fgi TRUE "" ${SRC})
+target_link_libraries(e3sm_fgi PRIVATE $<TARGET_OBJECTS:argparser>)
 
 if (WITH_HDF5)
   SET(SRC ${PROJECT_SOURCE_DIR}/../util/argparser.cxx hdf5.cpp)

examples/cxx/e3sm_f.cpp

@@ -0,0 +1,170 @@
+#include "e3sm_fgi_utils.hpp"
+#include "e3sm_fgi_data.hpp"
+
+/* Create a unique name for the test output */
+static inline std::string get_fcase_test_fname(int iotype, int rearr)
+{
+  const std::string FNAME_PREFIX = "spio_e3sm_fgi_f";
+  const std::string FNAME_SUFFIX = ".nc";
+  const std::string SEP = "_";
+
+  return FNAME_PREFIX
+          + SEP + Util::GVars::iotype2str(iotype)
+          + SEP + Util::GVars::rearr2str(rearr)
+          + FNAME_SUFFIX;
+}
+
+/* Pseudo E3SM F case */
+static int test_fcase(MPI_Comm comm, int iosysid, const std::string &fname, int iotype)
+{
+  int comm_rank = 0, comm_sz = 0;
+  int ret = PIO_NOERR;
+
+
+  ret = MPI_Comm_rank(comm, &comm_rank); assert(ret == MPI_SUCCESS);
+  ret = MPI_Comm_size(comm, &comm_sz); assert(ret == MPI_SUCCESS);
+
+  const int NE = 120;
+  const PIO_Offset NCOL_PER_PROC = 4;
+  const PIO_Offset NCOL = comm_sz * NCOL_PER_PROC;
+  const int NBND = 2;
+  const int MAX_CHARS = 8;
+  const int NLEV = 3;
+  const int NFRAMES = 2;
+  const bool is_last_proc = (comm_rank != comm_sz - 1) ? false : true;
+
+  int idx = 0, ilev = 0;
+  std::function<double(void )> range_zero_to_inf = [idx] (void ) mutable { return idx++; }; 
+
+  double dlev = 0;
+  std::function<double(void )> gen_levels = [dlev] (void ) mutable { double d = dlev; dlev += 0.2; return d; };
+
+  /* Generates dates for June 2021 starting from date 15/06/21 */
+  int start_date = 15;
+  std::function<std::string(void )> gen_dates = [start_date] (void ) mutable {
+    static const std::string month("06");
+    static const std::string year("21");
+    static const std::string SEP("/");
+
+    if(start_date >= 30) { start_date = 1; }
+
+    return month + SEP + std::to_string(start_date++) + SEP + year;
+  };
+
+  /* Generator for a simple 1D I/O decomposition */
+  std::function<PIO_Offset(void )> decomp_1d_gen =
+                          [comm_rank, NCOL, NCOL_PER_PROC](void ){
+                            static const PIO_Offset INIT_IDX = 0;
+                            static PIO_Offset idx = INIT_IDX;
+                            PIO_Offset val = comm_rank * NCOL_PER_PROC + idx++ + 1;
+
+                            if(idx >= NCOL) { idx = INIT_IDX; }
+
+                            return val;
+                          };
+
+  E3SM_FGI::SPIO_decomp decomp_1d("decomp_1d_ncol", iosysid, PIO_DOUBLE,
+                          std::vector<int>({static_cast<int>(NCOL)}), NCOL_PER_PROC, decomp_1d_gen);
+
+  /* 2D Decompisition generator : NCOL s are divided evenly among processes */
+  std::function<PIO_Offset(void ) > decomp_2d_gen =
+    [comm_rank, is_last_proc, NCOL, NCOL_PER_PROC, NLEV, idx, ilev](void ) mutable {
+
+      PIO_Offset val = ilev * NCOL + comm_rank * NCOL_PER_PROC + idx++ + 1;
+
+      if(!is_last_proc){
+        if(idx >= NCOL_PER_PROC) { idx = 0; ilev++; }
+      }
+      else{
+        if(idx >= (NCOL - comm_rank * NCOL_PER_PROC)) { idx = 0; ilev++; }
+      }
+
+      return val;
+    };
+
+  std::function<double(void) > decomp_2d_val_gen =
+    [decomp_2d_gen](void ){
+      return static_cast<double>(decomp_2d_gen());
+    };
+
+  int lsz = NLEV * ((!is_last_proc) ? NCOL_PER_PROC : (NCOL - comm_rank * NCOL_PER_PROC));
+
+  /* Create 2D I/O decomposition for unlimited vars */  
+  E3SM_FGI::SPIO_decomp decomp_2d("decomp_2d_lev_ncol", iosysid, PIO_DOUBLE,
+                          std::vector<int>({NLEV, static_cast<int>(NCOL)}),
+                          lsz, decomp_2d_gen);
+  std::shared_ptr<const E3SM_FGI::SPIO_decomp> pdecomp_2d =
+    std::make_shared<const E3SM_FGI::SPIO_decomp>(std::move(decomp_2d));
+
+  E3SM_FGI::SPIO_file fh(iosysid, fname, iotype, false);
+
+  /* Define dims/atts/vars */
+  ret = fh.def({
+            /* Global Attributes */
+            std::make_shared<E3SM_FGI::SPIO_att>("ne", NE),
+            std::make_shared<E3SM_FGI::SPIO_att>("title", "EAM History file information"),
+            std::make_shared<E3SM_FGI::SPIO_att>("source", "E3SM Atmosphere Model"),
+            std::make_shared<E3SM_FGI::SPIO_att>("empty_string", ""),
+            /* Dimensions */
+            std::make_shared<E3SM_FGI::SPIO_dim>("time", PIO_UNLIMITED),
+            std::make_shared<E3SM_FGI::SPIO_dim>("nbnd", NBND),
+            std::make_shared<E3SM_FGI::SPIO_dim>("chars", MAX_CHARS),
+            std::make_shared<E3SM_FGI::SPIO_dim>("lev", NLEV),
+            std::make_shared<E3SM_FGI::SPIO_dim>("ncol", NCOL),
+            /* Variables */
+            std::make_shared<E3SM_FGI::SPIO_var<double> >("lat", std::vector<std::string>({"ncol"}), std::vector<E3SM_FGI::SPIO_att>({{"long_name", "latitude"}, {"units", "degrees_north"}}), range_zero_to_inf),
+            std::make_shared<E3SM_FGI::SPIO_var<double> >("lev", std::vector<std::string>({"lev"}), std::vector<E3SM_FGI::SPIO_att>({{"long_name", "reference pressure"}, {"units", "Pa"}}), gen_levels),
+            std::make_shared<E3SM_FGI::SPIO_var<double> >("P0", std::vector<std::string>({}), std::vector<E3SM_FGI::SPIO_att>({}), std::vector<double>{99999.9999}),
+            std::make_shared<E3SM_FGI::SPIO_var<double> >("time", std::vector<std::string>({"time"}), std::vector<E3SM_FGI::SPIO_att>({}), range_zero_to_inf),
+            std::make_shared<E3SM_FGI::SPIO_cs_var<double> >("time_bnds", std::vector<std::string>({"time", "nbnd"}), std::vector<E3SM_FGI::SPIO_att>({}), std::vector<std::vector<PIO_Offset> > ({std::vector<PIO_Offset>({0, 0}), std::vector<PIO_Offset>({1, 0})}), std::vector<std::vector<PIO_Offset> > ({std::vector<PIO_Offset>({1, 2}), std::vector<PIO_Offset>({1, 2})}), range_zero_to_inf),
+            std::make_shared<E3SM_FGI::SPIO_cs_var<std::string> >("date_written", std::vector<std::string>({"time", "chars"}), std::vector<E3SM_FGI::SPIO_att>({}), std::vector<std::vector<PIO_Offset> > ({std::vector<PIO_Offset>({0, 0}), std::vector<PIO_Offset>({1, 0})}), std::vector<std::vector<PIO_Offset> > ({std::vector<PIO_Offset>({1, MAX_CHARS}), std::vector<PIO_Offset>({1, MAX_CHARS})}), gen_dates),
+            std::make_shared<E3SM_FGI::SPIO_unlimited_var<double> >("U", std::vector<std::string>({"time", "lev", "ncol"}), std::vector<E3SM_FGI::SPIO_att>({}), pdecomp_2d, NFRAMES, decomp_2d_val_gen)
+          });
+  if(ret != PIO_NOERR){
+    Util::GVars::logger->log(Util::Logging::LogLevel::ERROR, "Defining file objects failed");
+    return ret;
+  }
+
+  /* Write variables/atts */
+  ret = fh.put();
+  if(ret != PIO_NOERR){
+    Util::GVars::logger->log(Util::Logging::LogLevel::ERROR, "Putting file objects failed");
+    return ret;
+  }
+
+  return ret;
+}
+
+int E3SM_FGI::test_e3sm_fcase(MPI_Comm comm, const std::vector<int> &iotypes,
+      const std::vector<int> &rearrs, int nioprocs)
+{
+  int comm_rank, comm_sz, ret;
+  ret = MPI_Comm_rank(comm, &comm_rank); assert(ret == MPI_SUCCESS);
+  ret = MPI_Comm_size(comm, &comm_sz); assert(ret == MPI_SUCCESS);
+
+  assert(nioprocs <= comm_sz);
+
+  int io_stride = comm_sz / nioprocs;
+
+  /* Test F case for each combination of I/O type and I/O rearranger */
+  Util::zip_for_each(iotypes.cbegin(), iotypes.cend(), rearrs.cbegin(), rearrs.cend(),
+    [comm, nioprocs, io_stride](int iotype, int rearr){
+      const int IOSYS_START_PROC = 0;
+      const int SPIO_IOSYSID_INVALID = -2;
+      int ret = PIO_NOERR;
+
+      Util::GVars::logger->log(Util::Logging::LogLevel::STATUS, "Testing F Case, iotype = " + Util::GVars::iotype2str(iotype) + ", rearr = " + Util::GVars::rearr2str(rearr) + "\n");
+      int iosysid = SPIO_IOSYSID_INVALID;
+      ret = PIOc_Init_Intracomm(comm, nioprocs, io_stride, IOSYS_START_PROC,
+              rearr, &iosysid);
+      Util::check_spio_err(ret, "PIOc_Init_Intracomm failed", __FILE__, __LINE__);
+
+      ret = test_fcase(comm, iosysid, get_fcase_test_fname(iotype, rearr), iotype);
+      Util::check_spio_err(ret, "Testing F case failed", __FILE__, __LINE__);
+
+      PIOc_finalize(iosysid);
+    });
+
+  return PIO_NOERR;
+}
+