diff --git a/CMakeLists.txt b/CMakeLists.txt index 9c013b5120..a6e4218422 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -66,9 +66,17 @@ option (HDF5_USE_LOSSY_COMPRESSION "Enable lossy compression methods in HDF5 (e option (PIO_ENABLE_TESTS "Enable the testing builds" OFF) option (PIO_ENABLE_LARGE_TESTS "Enable large (file, processes) tests" OFF) option (PIO_VALGRIND_CHECK "Enable memory leak check using valgrind" OFF) -#===== Dependent Options ===== -include(CMakeDependentOption) -cmake_dependent_option (PIO_TEST_CLOSE_OPEN_FOR_SYNC "SCORPIO fortran tests will close+open for sync" ON "WITH_ADIOS2" OFF) +option (PIO_TEST_CLOSE_OPEN_FOR_SYNC "Enable close+open to sync file output in tests" OFF) + +if (WITH_ADIOS2) + message(STATUS "Enabling close+open to sync file output for SCORPIO tests (default since ADIOS is enabled)") + set(PIO_TEST_CLOSE_OPEN_FOR_SYNC ON CACHE BOOL "Enable close+open to sync file output in tests" FORCE) +endif () + +if (WITH_HDF5 AND (NOT PIO_TEST_CLOSE_OPEN_FOR_SYNC)) + message(STATUS "Enabling close+open to sync file output for SCORPIO tests (default since HDF5 is enabled)") + set(PIO_TEST_CLOSE_OPEN_FOR_SYNC ON CACHE BOOL "Enable close+open to sync file output in tests" FORCE) +endif () # Get the name of the build host cmake_host_system_information (RESULT FQDN_SITENAME QUERY FQDN) @@ -229,6 +237,20 @@ else() message(STATUS "Reserving some extra space in the header when creating NetCDF files, requested bytes = " ${PIO_RESERVED_FILE_HEADER_SIZE} " (default)") endif() +# Set chunk size (in bytes) for HDF5/PnetCDF chunked variables. Default is 4 MB, same as NetCDF4. +set(DEF_SPIO_CHUNK_SIZE 4194304) +if(DEFINED PIO_CHUNK_SIZE) + if(PIO_CHUNK_SIZE GREATER_EQUAL 1) + message(STATUS "Setting chunk size (in bytes) for HDF5/PnetCDF chunked variables, requested bytes = " ${PIO_CHUNK_SIZE}) + else() + message(WARNING "User-defined PIO_CHUNK_SIZE is invalid, setting it to " ${DEF_SPIO_CHUNK_SIZE} " (default)") + set(PIO_CHUNK_SIZE ${DEF_SPIO_CHUNK_SIZE}) + endif() +else() + set(PIO_CHUNK_SIZE ${DEF_SPIO_CHUNK_SIZE}) + message(STATUS "Setting chunk size (in bytes) for HDF5/PnetCDF chunked variables, requested bytes = " ${PIO_CHUNK_SIZE} " (default)") +endif() + if(WITH_ADIOS2) # Maximum number of I/O decompositions registered with ADIOS type set(DEF_SPIO_MAX_ADIOS_DECOMPS 65536) diff --git a/examples/c/test_hdf5.c b/examples/c/test_hdf5.c index d8e9f92ecf..a3d7e767bd 100644 --- a/examples/c/test_hdf5.c +++ b/examples/c/test_hdf5.c @@ -43,6 +43,12 @@ int main(int argc, char* argv[]) int dimid_ncol; int dimid_sample; int dimid_ship; + int dimid_nCells; + int dimid_nEdges; + int dimid_nVertices; + int dimid_nVertLevels; + int dimid_nVertLevelsP1; + int dimid_StrLen; int varid; int varid_time; @@ -57,6 +63,7 @@ int main(int argc, char* argv[]) int varid_U; int varid_sample; int varid_ship; + int varid_normalVelocity; int att_id; int att_type; @@ -594,6 +601,86 @@ int main(int argc, char* argv[]) ret = PIOc_closefile(ncid); ERR + if (my_rank == 0) + { + printf("Test reading %s with NETCDF4 IO type end\n", filename); + fflush(stdout); + } +#endif + + /* Simulate a history file of E3SM G case with large dimension lengths (compset GMPAS-NYF, res T62_oRRS18to6v3) */ + snprintf(filename, PIO_MAX_NAME, "test_hdf5_g_case_rearr_%d.nc", rearranger[r]); + + if (my_rank == 0) + { + printf("Test writing %s with HDF5 IO type start\n", filename); + fflush(stdout); + } + + ncid = -1; + ret = PIOc_createfile(iosysid, &ncid, &format, filename, PIO_CLOBBER); ERR + + ret = PIOc_def_dim(ncid, "Time", PIO_UNLIMITED, &dimid_time); ERR + ret = PIOc_def_dim(ncid, "nCells", 3693225, &dimid_nCells); ERR + ret = PIOc_def_dim(ncid, "nEdges", 11135652, &dimid_nEdges); ERR + ret = PIOc_def_dim(ncid, "nVertices", 7441216, &dimid_nVertices); ERR + ret = PIOc_def_dim(ncid, "nVertLevels", 81, &dimid_nVertLevels); ERR + ret = PIOc_def_dim(ncid, "nVertLevelsP1", 81, &dimid_nVertLevelsP1); ERR + ret = PIOc_def_dim(ncid, "StrLen", 64, &dimid_StrLen); ERR + + dimids[0] = dimid_time; + dimids[1] = dimid_nEdges; + dimids[2] = dimid_nVertLevels; + ret = PIOc_def_var(ncid, "normalVelocity", PIO_DOUBLE, 3, dimids, &varid_normalVelocity); ERR + + ret = PIOc_enddef(ncid); ERR + + ret = PIOc_closefile(ncid); ERR + + if (my_rank == 0) + { + printf("Test writing %s with HDF5 IO type end\n", filename); + fflush(stdout); + } + +#ifdef _NETCDF4 + /* Direct read support for HDF5 IO type is not implemented yet: SCORPIO implicitly switches HDF5 type to + * NETCDF4 type for reading back output files (generated by HDF5 type, fully NETCDF4 compatible). */ + if (my_rank == 0) + { + printf("Test reading %s with NETCDF4 IO type start\n", filename); + fflush(stdout); + } + + ncid = -1; + ret = PIOc_openfile(iosysid, &ncid, &format, filename, PIO_NOWRITE); ERR + + ndims = -1; + ret = PIOc_inq_ndims(ncid, &ndims); ERR + assert(ndims == 7); + + dimid_time = -1; + ret = PIOc_inq_dimid(ncid, "Time", &dimid_time); ERR + assert(dimid_time >= 0 && dimid_time < ndims); + + nvars = -1; + ret = PIOc_inq_nvars(ncid, &nvars); ERR + assert(nvars == 1); + + PIO_Offset dimlen_nCells = -1; + ret = PIOc_inq_dimlen(ncid, dimid_nCells, &dimlen_nCells); ERR + assert(dimlen_nCells == 3693225); + + PIO_Offset dimlen_nEdges = -1; + ret = PIOc_inq_dimlen(ncid, dimid_nEdges, &dimlen_nEdges); ERR + assert(dimlen_nEdges == 11135652); + + PIO_Offset dimlen_nVertices = -1; + ret = PIOc_inq_dimlen(ncid, dimid_nVertices, &dimlen_nVertices); ERR + assert(dimlen_nVertices == 7441216); + + ret = PIOc_closefile(ncid); ERR + if (my_rank == 0) { printf("Test reading %s with NETCDF4 IO type end\n", filename); diff --git a/examples/cxx/CMakeLists.txt b/examples/cxx/CMakeLists.txt index 800a6f3b82..1bbc4c2384 100644 --- a/examples/cxx/CMakeLists.txt +++ b/examples/cxx/CMakeLists.txt @@ -48,8 +48,16 @@ LINK_DIRECTORIES(${PIO_LIB_DIR}) #============================================================================== # BUILD EXECUTABLE #============================================================================== +SET(SRC ${SCORPIO_SOURCE_DIR}/util/argparser.cxx) +add_library(argparser OBJECT ${SRC}) + SET(SRC examplePio.cxx) add_spio_executable(examplePio_cxx TRUE "" ${SRC}) +target_link_libraries(examplePio_cxx PRIVATE $) + +SET(SRC e3sm_fgi.cpp e3sm_f.cpp e3sm_g.cpp e3sm_i.cpp) +add_spio_executable(e3sm_fgi TRUE "" ${SRC}) +target_link_libraries(e3sm_fgi PRIVATE $) if (WITH_HDF5) SET(SRC ${PROJECT_SOURCE_DIR}/../util/argparser.cxx hdf5.cpp) diff --git a/examples/cxx/e3sm_f.cpp b/examples/cxx/e3sm_f.cpp new file mode 100644 index 0000000000..b42a333a11 --- /dev/null +++ b/examples/cxx/e3sm_f.cpp @@ -0,0 +1,170 @@ +#include "e3sm_fgi_utils.hpp" +#include "e3sm_fgi_data.hpp" + +/* Create a unique name for the test output */ +static inline std::string get_fcase_test_fname(int iotype, int rearr) +{ + const std::string FNAME_PREFIX = "spio_e3sm_fgi_f"; + const std::string FNAME_SUFFIX = ".nc"; + const std::string SEP = "_"; + + return FNAME_PREFIX + + SEP + Util::GVars::iotype2str(iotype) + + SEP + Util::GVars::rearr2str(rearr) + + FNAME_SUFFIX; +} + +/* Pseudo E3SM F case */ +static int test_fcase(MPI_Comm comm, int iosysid, const std::string &fname, int iotype) +{ + int comm_rank = 0, comm_sz = 0; + int ret = PIO_NOERR; + + + ret = MPI_Comm_rank(comm, &comm_rank); assert(ret == MPI_SUCCESS); + ret = MPI_Comm_size(comm, &comm_sz); assert(ret == MPI_SUCCESS); + + const int NE = 120; + const PIO_Offset NCOL_PER_PROC = 4; + const PIO_Offset NCOL = comm_sz * NCOL_PER_PROC; + const int NBND = 2; + const int MAX_CHARS = 8; + const int NLEV = 3; + const int NFRAMES = 2; + const bool is_last_proc = (comm_rank != comm_sz - 1) ? false : true; + + int idx = 0, ilev = 0; + std::function range_zero_to_inf = [idx] (void ) mutable { return idx++; }; + + double dlev = 0; + std::function gen_levels = [dlev] (void ) mutable { double d = dlev; dlev += 0.2; return d; }; + + /* Generates dates for June 2021 starting from date 15/06/21 */ + int start_date = 15; + std::function gen_dates = [start_date] (void ) mutable { + static const std::string month("06"); + static const std::string year("21"); + static const std::string SEP("/"); + + if(start_date >= 30) { start_date = 1; } + + return month + SEP + std::to_string(start_date++) + SEP + year; + }; + + /* Generator for a simple 1D I/O decomposition */ + std::function decomp_1d_gen = + [comm_rank, NCOL, NCOL_PER_PROC](void ){ + static const PIO_Offset INIT_IDX = 0; + static PIO_Offset idx = INIT_IDX; + PIO_Offset val = comm_rank * NCOL_PER_PROC + idx++ + 1; + + if(idx >= NCOL) { idx = INIT_IDX; } + + return val; + }; + + E3SM_FGI::SPIO_decomp decomp_1d("decomp_1d_ncol", iosysid, PIO_DOUBLE, + std::vector({static_cast(NCOL)}), NCOL_PER_PROC, decomp_1d_gen); + + /* 2D Decompisition generator : NCOL s are divided evenly among processes */ + std::function decomp_2d_gen = + [comm_rank, is_last_proc, NCOL, NCOL_PER_PROC, NLEV, idx, ilev](void ) mutable { + + PIO_Offset val = ilev * NCOL + comm_rank * NCOL_PER_PROC + idx++ + 1; + + if(!is_last_proc){ + if(idx >= NCOL_PER_PROC) { idx = 0; ilev++; } + } + else{ + if(idx >= (NCOL - comm_rank * NCOL_PER_PROC)) { idx = 0; ilev++; } + } + + return val; + }; + + std::function decomp_2d_val_gen = + [decomp_2d_gen](void ){ + return static_cast(decomp_2d_gen()); + }; + + int lsz = NLEV * ((!is_last_proc) ? NCOL_PER_PROC : (NCOL - comm_rank * NCOL_PER_PROC)); + + /* Create 2D I/O decomposition for unlimited vars */ + E3SM_FGI::SPIO_decomp decomp_2d("decomp_2d_lev_ncol", iosysid, PIO_DOUBLE, + std::vector({NLEV, static_cast(NCOL)}), + lsz, decomp_2d_gen); + std::shared_ptr pdecomp_2d = + std::make_shared(std::move(decomp_2d)); + + E3SM_FGI::SPIO_file fh(iosysid, fname, iotype, false); + + /* Define dims/atts/vars */ + ret = fh.def({ + /* Global Attributes */ + std::make_shared("ne", NE), + std::make_shared("title", "EAM History file information"), + std::make_shared("source", "E3SM Atmosphere Model"), + std::make_shared("empty_string", ""), + /* Dimensions */ + std::make_shared("time", PIO_UNLIMITED), + std::make_shared("nbnd", NBND), + std::make_shared("chars", MAX_CHARS), + std::make_shared("lev", NLEV), + std::make_shared("ncol", NCOL), + /* Variables */ + std::make_shared >("lat", std::vector({"ncol"}), std::vector({{"long_name", "latitude"}, {"units", "degrees_north"}}), range_zero_to_inf), + std::make_shared >("lev", std::vector({"lev"}), std::vector({{"long_name", "reference pressure"}, {"units", "Pa"}}), gen_levels), + std::make_shared >("P0", std::vector({}), std::vector({}), std::vector{99999.9999}), + std::make_shared >("time", std::vector({"time"}), std::vector({}), range_zero_to_inf), + std::make_shared >("time_bnds", std::vector({"time", "nbnd"}), std::vector({}), std::vector > ({std::vector({0, 0}), std::vector({1, 0})}), std::vector > ({std::vector({1, 2}), std::vector({1, 2})}), range_zero_to_inf), + std::make_shared >("date_written", std::vector({"time", "chars"}), std::vector({}), std::vector > ({std::vector({0, 0}), std::vector({1, 0})}), std::vector > ({std::vector({1, MAX_CHARS}), std::vector({1, MAX_CHARS})}), gen_dates), + std::make_shared >("U", std::vector({"time", "lev", "ncol"}), std::vector({}), pdecomp_2d, NFRAMES, decomp_2d_val_gen) + }); + if(ret != PIO_NOERR){ + Util::GVars::logger->log(Util::Logging::LogLevel::ERROR, "Defining file objects failed"); + return ret; + } + + /* Write variables/atts */ + ret = fh.put(); + if(ret != PIO_NOERR){ + Util::GVars::logger->log(Util::Logging::LogLevel::ERROR, "Putting file objects failed"); + return ret; + } + + return ret; +} + +int E3SM_FGI::test_e3sm_fcase(MPI_Comm comm, const std::vector &iotypes, + const std::vector &rearrs, int nioprocs) +{ + int comm_rank, comm_sz, ret; + ret = MPI_Comm_rank(comm, &comm_rank); assert(ret == MPI_SUCCESS); + ret = MPI_Comm_size(comm, &comm_sz); assert(ret == MPI_SUCCESS); + + assert(nioprocs <= comm_sz); + + int io_stride = comm_sz / nioprocs; + + /* Test F case for each combination of I/O type and I/O rearranger */ + Util::zip_for_each(iotypes.cbegin(), iotypes.cend(), rearrs.cbegin(), rearrs.cend(), + [comm, nioprocs, io_stride](int iotype, int rearr){ + const int IOSYS_START_PROC = 0; + const int SPIO_IOSYSID_INVALID = -2; + int ret = PIO_NOERR; + + Util::GVars::logger->log(Util::Logging::LogLevel::STATUS, "Testing F Case, iotype = " + Util::GVars::iotype2str(iotype) + ", rearr = " + Util::GVars::rearr2str(rearr) + "\n"); + int iosysid = SPIO_IOSYSID_INVALID; + ret = PIOc_Init_Intracomm(comm, nioprocs, io_stride, IOSYS_START_PROC, + rearr, &iosysid); + Util::check_spio_err(ret, "PIOc_Init_Intracomm failed", __FILE__, __LINE__); + + ret = test_fcase(comm, iosysid, get_fcase_test_fname(iotype, rearr), iotype); + Util::check_spio_err(ret, "Testing F case failed", __FILE__, __LINE__); + + PIOc_finalize(iosysid); + }); + + return PIO_NOERR; +} + diff --git a/examples/cxx/e3sm_fgi.cpp b/examples/cxx/e3sm_fgi.cpp new file mode 100644 index 0000000000..6edc041412 --- /dev/null +++ b/examples/cxx/e3sm_fgi.cpp @@ -0,0 +1,231 @@ +#include "e3sm_fgi_utils.hpp" +#include +#include +#include "argparser.h" + +namespace Util{ + namespace GVars{ + /* Types used for parsing user input */ + /* Available I/O types */ + std::unordered_map iotypes = { + {"PNETCDF", PIO_IOTYPE_PNETCDF}, + {"NETCDF", PIO_IOTYPE_NETCDF}, + {"NETCDF4C", PIO_IOTYPE_NETCDF4C}, + {"NETCDF4P", PIO_IOTYPE_NETCDF4P}, + {"NCZARR", PIO_IOTYPE_NETCDF4P_NCZARR}, + {"HDF5", PIO_IOTYPE_HDF5}, + {"HDF5C", PIO_IOTYPE_HDF5C} + }; + + /* Available I/O rearrangers */ + std::unordered_map rearrs = { + {"SUBSET", PIO_REARR_SUBSET}, + {"BOX", PIO_REARR_BOX}, + {"ANY", PIO_REARR_ANY} + }; + + /* E3SM pseudo test cases */ + std::unordered_map cases = { + {"F", E3SM_FGI::Case_Type::E3SM_F_CASE}, + {"G", E3SM_FGI::Case_Type::E3SM_G_CASE}, + {"I", E3SM_FGI::Case_Type::E3SM_I_CASE} + }; + + /* Log levels */ + std::unordered_map llevels = { + {"STATUS", Util::Logging::LogLevel::STATUS}, + {"VERBOSE", Util::Logging::LogLevel::VERBOSE}, + {"WARNING", Util::Logging::LogLevel::WARNING}, + {"ERROR", Util::Logging::LogLevel::ERROR} + }; + + /* The global logger */ + std::shared_ptr logger; + } +} + +/* Initialize the argparser options - specify the available user opts */ +static void init_user_options(spio_tool_utils::ArgParser &ap) +{ + ap.add_opt("pio-format", "SCORPIO I/O type (for output data). Supported iotypes: " + Util::GVars::iotypes2str()) + .add_opt("nioprocs", "Number of I/O processes") + .add_opt("rearr", "SCORPIO rearranger. Supported rearrangers: " + Util::GVars::rearrs2str()) + .add_opt("e3sm-case", "Pseudo E3SM Cases to run: " + Util::GVars::cases2str()) + .add_opt("log-level", "Logging level : " + Util::GVars::llevels2str()) + .add_opt("help", "Print help"); +} + +/* Parse the user options */ +static int get_user_options( + spio_tool_utils::ArgParser &ap, + int argc, char *argv[], + std::vector &iotypes, + std::vector &rearrs, + std::vector &cases, + Util::Logging::LogLevel &llevel, + int nioprocs, int rank) +{ +#ifdef SPIO_NO_CXX_REGEX + ap.no_regex_parse(argc, argv); +#else + ap.parse(argc, argv); +#endif + + if(ap.has_arg("help")){ + return -1; + } + + std::string iotype_str, rearr_str, ctype_str, llevel_str; + if(ap.has_arg("pio-format")){ + iotype_str = Util::String::toupper(ap.get_arg("pio-format")); + try{ + iotypes.push_back(Util::GVars::str2iotype(iotype_str)); + } + catch(std::out_of_range &e){ + Util::GVars::logger->log(Util::Logging::LogLevel::ERROR, "Invalid I/O format specified : " + iotype_str); + Util::GVars::logger->log(Util::Logging::LogLevel::ERROR, e.what()); + ap.print_usage(std::cerr); + throw; + } + } + else{ + Util::GVars::copy_opt_map(Util::GVars::iotypes, std::back_inserter(iotypes)); + } + + if(ap.has_arg("rearr")){ + rearr_str = Util::String::toupper(ap.get_arg("rearr")); + try{ + rearrs.push_back(Util::GVars::str2rearr(rearr_str)); + } + catch(std::out_of_range &e){ + Util::GVars::logger->log(Util::Logging::LogLevel::ERROR, "Invalid Rearranger specified : " + rearr_str); + Util::GVars::logger->log(Util::Logging::LogLevel::ERROR, e.what()); + ap.print_usage(std::cerr); + throw; + } + } + else{ + Util::GVars::copy_opt_map(Util::GVars::rearrs, std::back_inserter(rearrs)); + } + + if(ap.has_arg("e3sm-case")){ + ctype_str = Util::String::toupper(ap.get_arg("e3sm-case")); + try{ + cases.push_back(Util::GVars::str2case(ctype_str)); + } + catch(std::out_of_range &e){ + Util::GVars::logger->log(Util::Logging::LogLevel::ERROR, "Invalid E3SM case specified : " + ctype_str); + Util::GVars::logger->log(Util::Logging::LogLevel::ERROR, e.what()); + ap.print_usage(std::cerr); + throw; + } + } + else{ + Util::GVars::copy_opt_map(Util::GVars::cases, std::back_inserter(cases)); + } + + if(ap.has_arg("log-level")){ + llevel_str = Util::String::toupper(ap.get_arg("log-level")); + try{ + llevel = Util::GVars::str2llevel(llevel_str); + } + catch(std::out_of_range &e){ + Util::GVars::logger->log(Util::Logging::LogLevel::ERROR, "Invalid logging level specified : " + llevel_str); + Util::GVars::logger->log(Util::Logging::LogLevel::ERROR, e.what()); + ap.print_usage(std::cerr); + throw; + } + } + else{ + llevel = Util::Logging::LogLevel::STATUS; + } + + nioprocs = 1; + if(ap.has_arg("nioprocs")){ + nioprocs = ap.get_arg("nioprocs"); + if(nioprocs <= 0){ + Util::GVars::logger->log(Util::Logging::LogLevel::ERROR, "Invalid number of I/O processes specified : " + std::to_string(nioprocs)); + throw std::runtime_error("Command line argument parse error"); + } + } + + return 0; +} + +int main(int argc, char *argv[]) +{ + int ret = 0, rank = 0; + + MPI_Init(&argc, &argv); + + Util::GVars::logger = Util::Logging::Logger::get_logger(MPI_COMM_WORLD, "STDOUT"); + Util::GVars::logger->set_log_level(Util::Logging::LogLevel::STATUS); + + MPI_Comm comm_in = MPI_COMM_WORLD; + ret = MPI_Comm_rank(comm_in, &rank); assert(ret == MPI_SUCCESS); + + spio_tool_utils::ArgParser ap(comm_in); + + /* Init the standard user options for the tool */ + init_user_options(ap); + + /* Parse the user options */ + std::vector iotypes, rearrs; + std::vector cases; + Util::Logging::LogLevel log_lvl; + int nioprocs = 1; + ret = get_user_options(ap, argc, argv, iotypes, rearrs, cases, log_lvl, nioprocs, rank); + if(ret != 0){ + MPI_Finalize(); + return ret; + } + + /* Only log from rank 0 */ + Util::GVars::logger->set_log_rank(0); + Util::GVars::logger->set_log_level(log_lvl); + + MPI_Barrier(comm_in); +#ifdef SPIO_ENABLE_GPTL_TIMING + /* Initialize the GPTL timing library. */ + if((ret = GPTLinitialize())){ + return ret; + } + + GPTLstart("e3sm_case:main"); +#endif + + for(std::vector::const_iterator citer = cases.cbegin(); + citer != cases.end(); ++citer){ + switch(*citer){ + case E3SM_FGI::Case_Type::E3SM_F_CASE : ret = E3SM_FGI::test_e3sm_fcase(comm_in, iotypes, rearrs, nioprocs); break; + case E3SM_FGI::Case_Type::E3SM_G_CASE : ret = E3SM_FGI::test_e3sm_gcase(comm_in, iotypes, rearrs, nioprocs); break; + case E3SM_FGI::Case_Type::E3SM_I_CASE : ret = E3SM_FGI::test_e3sm_icase(comm_in, iotypes, rearrs, nioprocs); break; + } + if(ret != PIO_NOERR){ + Util::GVars::logger->log(Util::Logging::LogLevel::ERROR, "Test failed, case=" + Util::GVars::case2str(*citer) + ", ret = " + std::to_string(ret)); + } + } + + MPI_Barrier(comm_in); +#ifdef SPIO_ENABLE_GPTL_TIMING + GPTLstop("e3sm_case:main"); + + /* Write the GPTL summary/rank_0 output */ + std::string summary_file("spioe3smfgirwgptlsummaryinfo0wrank.dat"); + GPTLpr_summary_file(comm_in, summary_file.c_str()); + + if(rank == 0){ + std::string rank0_file("spioe3smfgirwgptlinfo0wrank.dat"); + GPTLpr_file(rank0_file.c_str()); + } + + /* Finalize the GPTL timing library. */ + if((ret = GPTLfinalize())){ + return ret; + } +#endif + + MPI_Finalize(); + + return ret; +} diff --git a/examples/cxx/e3sm_fgi_data.hpp b/examples/cxx/e3sm_fgi_data.hpp new file mode 100644 index 0000000000..43b418d9aa --- /dev/null +++ b/examples/cxx/e3sm_fgi_data.hpp @@ -0,0 +1,1000 @@ +#ifndef __E3SM_FGI_DATA_HPP__ +#define __E3SM_FGI_DATA_HPP__ + +#include "e3sm_fgi_utils.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace E3SM_FGI{ + class SPIO_file; + + /* Interface for classes that have a name & id */ + class IHas_name_id{ + public: + virtual std::string name(void ) const = 0; + virtual int id(void ) const = 0; + virtual ~IHas_name_id() = default; + }; + + /* All objects added to a file are "File Objects" */ + class SPIO_file_obj : public IHas_name_id{ + public: + virtual int def(SPIO_file &f) = 0; + virtual int put(void ) = 0; + virtual int get_and_verify(void ) const = 0; + virtual ~SPIO_file_obj() = default; + }; + + /* A valid file that has a valid name/id */ + class SPIO_valid_file : public IHas_name_id{ + public: + virtual ~SPIO_valid_file() = default; + }; + /* A valid dim that has a valid name/id */ + class SPIO_valid_dim : public SPIO_file_obj{ + public: + virtual ~SPIO_valid_dim() = default; + }; + /* A valid var that has a valid name/id */ + class SPIO_valid_var : public SPIO_file_obj{ + public: + virtual ~SPIO_valid_var() = default; + }; + + template + class SPIO_var; + class SPIO_dim; + + /* SPIO I/O decomposition class. I/O decompositions are used to specify how + * data is decomposed across MPI processes for variables with unlimited + * dimensions (SPIO_unlimited_var) + * The SCORPIO I/O decomposition is created in the constructor and freed + * in the desctructor. + */ + class SPIO_decomp : public IHas_name_id{ + public: + SPIO_decomp(const std::string &name, int iosysid, int pio_type, + const std::vector &gdim_sz, + int lsz, std::function &gen): + name_(name), iosysid_(iosysid), pio_type_(pio_type), + gdim_sz_(gdim_sz), lsz_(lsz), id_(INVALID_ID) + { + int ret = PIO_NOERR; + assert((iosysid >= 0) && gen); + std::vector decomp_map; + + Util::GVars::logger->log(Util::Logging::LogLevel::VERBOSE, "Creating decomp : " + name_ + "\n"); + + /* Generate the I/O decomposition */ + decomp_map.reserve(lsz); + std::generate_n(std::back_inserter(decomp_map), lsz, gen); + + ret = PIOc_InitDecomp(iosysid, pio_type, gdim_sz.size(), gdim_sz.data(), + lsz, decomp_map.data(), &id_, NULL, NULL, NULL); + if(ret != PIO_NOERR){ + std::string err_msg("PIOc_InitDecomp() failed : "); + err_msg += "iosysid = " + std::to_string(iosysid) + + ", pio_type = " + std::to_string(pio_type) + + ", gdim_sz = " + Util::String::vec_to_string(gdim_sz) + + ", lsz = " + std::to_string(lsz) + "\n"; + + Util::GVars::logger->log(Util::Logging::LogLevel::ERROR, err_msg); + throw std::runtime_error(err_msg); + } + } + + SPIO_decomp(const SPIO_decomp &other) = delete; + /* I/O decompositions should not be copied over */ + SPIO_decomp &operator=(const SPIO_decomp &other) = delete; + /* Allow "moving" an I/O decompisition */ + SPIO_decomp(SPIO_decomp &&other) : + name_(std::move(other.name_)), iosysid_(other.iosysid_), pio_type_(other.pio_type_), + gdim_sz_(std::move(other.gdim_sz_)), lsz_(other.lsz_), id_(other.id_) + { + /* Invalidate other */ + other.id_ = INVALID_ID; + } + + std::string name(void ) const { return name_; } + int id(void ) const { return id_; } + + ~SPIO_decomp() + { + int ret = PIO_NOERR; + if(has_valid_decomp()){ + Util::GVars::logger->log(Util::Logging::LogLevel::VERBOSE, "Freeing decomp : " + name_ + "\n"); + ret = PIOc_freedecomp(iosysid_, id_); + if(ret != PIO_NOERR){ + std::string err_msg("PIOc_freedecomp() failed :"); + err_msg += "iosysid = " + std::to_string(iosysid_) + + ", pio_type = " + std::to_string(pio_type_) + + ", gdim_sz = " + Util::String::vec_to_string(gdim_sz_) + + ", lsz = " + std::to_string(lsz_) + "\n"; + + Util::GVars::logger->log(Util::Logging::LogLevel::ERROR, err_msg); + } + } + } + private: + static const int INVALID_ID = -1; + + /* Name of the I/O decomposition */ + const std::string name_; + int iosysid_; + int pio_type_; + /* Global dimensions of the I/O decomposition */ + std::vector gdim_sz_; + /* Local size of the I/O decomposition */ + int lsz_; + /* Decomposition ID */ + int id_; + + bool has_valid_decomp(void ) const { return id_ > 0; } + }; + + /* Some traits util classes to convert between C++ and PIO types */ + namespace Type_Traits{ + template + struct cxx_to_pio_type : std::false_type {}; + + template<> + struct cxx_to_pio_type { static constexpr int pio_type = PIO_CHAR; }; + + template<> + struct cxx_to_pio_type { static constexpr int pio_type = PIO_CHAR; }; + + template<> + struct cxx_to_pio_type { static constexpr int pio_type = PIO_INT; }; + + template<> + struct cxx_to_pio_type { static constexpr int pio_type = PIO_FLOAT; }; + + template<> + struct cxx_to_pio_type { static constexpr int pio_type = PIO_DOUBLE; }; + + } // namespace TypeTraits + + /* A File class + * The SCORPIO File is created in the constructor and closed/deleted in the destructor */ + class SPIO_file : public SPIO_valid_file{ + public: + SPIO_file(int iosysid, const std::string &name, int iotype) : iosysid_(iosysid), + name_(name), iotype_(iotype), fh_(INVALID_ID), is_in_def_mode_(false), + del_on_close_(true){ + int ret = PIO_NOERR; + + Util::GVars::logger->log(Util::Logging::LogLevel::VERBOSE, "Creating file : " + name_ + "\n"); + ret = create_file(iosysid, name, iotype); + if(ret == PIO_NOERR){ + is_in_def_mode_ = true; + } + else{ + Util::GVars::logger->log(Util::Logging::LogLevel::ERROR, "Creating file : " + name + "failed\n"); + } + } + + SPIO_file(int iosysid, const std::string &name, int iotype, bool del_on_close) : + iosysid_(iosysid), name_(name), iotype_(iotype), fh_(INVALID_ID), + is_in_def_mode_(false), del_on_close_(del_on_close){ + int ret = PIO_NOERR; + + Util::GVars::logger->log(Util::Logging::LogLevel::VERBOSE, "Creating file : " + name_ + "\n"); + ret = create_file(iosysid, name, iotype); + if(ret == PIO_NOERR){ + is_in_def_mode_ = true; + } + else{ + Util::GVars::logger->log(Util::Logging::LogLevel::ERROR, "Creating file : " + name + "failed\n"); + } + } + + std::string name(void ) const { return name_; } + int id(void ) const { return fh_; } + int fh(void ) const { return id(); } + + /* Define all the file objects - dims/vars/atts - associated with this file */ + int def(const std::vector > &fobjs){ + int ret = PIO_NOERR; + assert(fh_ != INVALID_ID); + if(!is_in_def_mode_){ + ret = PIOc_redef(fh_); + if(ret != PIO_NOERR){ + std::string err_msg("PIOc_redef("); + err_msg += "file = " + name_ + + ") failed, ret = " + std::to_string(ret) + "\n"; + Util::GVars::logger->log(Util::Logging::LogLevel::ERROR, err_msg); + return ret; + } + } + + Util::GVars::logger->log(Util::Logging::LogLevel::VERBOSE, "Defining file objects in file : " + name_ + "\n"); + for(std::vector >::const_iterator citer = fobjs.cbegin(); + citer != fobjs.cend(); ++citer){ + ret = (*citer)->def(*this); + if(ret != PIO_NOERR){ + Util::GVars::logger->log(Util::Logging::LogLevel::ERROR, "Defining file object failed\n"); + return ret; + } + + /* Store the file object in an internal cache */ + dcache_[(*citer)->name()] = *citer; + } + + ret = PIOc_enddef(fh_); + if(ret != PIO_NOERR){ + std::string err_msg("PIOc_enddef("); + err_msg += "file = " + name_ + + ") failed, ret = " + std::to_string(ret) + "\n"; + Util::GVars::logger->log(Util::Logging::LogLevel::ERROR, err_msg); + return ret; + } + + is_in_def_mode_ = false; + + return PIO_NOERR; + } + + /* Put the file objects - vars/atts - associated with this file */ + int put(void ){ + int ret = PIO_NOERR; + assert(fh_ != INVALID_ID); + assert(!is_in_def_mode_); + + Util::GVars::logger->log(Util::Logging::LogLevel::VERBOSE, "Writing file objects in file : " + name_ + "\n"); + for(std::map >::const_iterator citer = dcache_.cbegin(); + citer != dcache_.cend(); ++citer){ + ret = citer->second->put(); + if(ret != PIO_NOERR){ + Util::GVars::logger->log(Util::Logging::LogLevel::ERROR, "Put/write of variable failed\n"); + return ret; + } + } + return PIO_NOERR; + } + + /* Get a pointer to the cached file object */ + std::shared_ptr get(const std::string &name) const{ + return dcache_.at(name); + } + + /* FIXME: Implement reading/verifying file objects */ + int get_and_verify(void ) { return PIO_NOERR; } + + ~SPIO_file(){ + if(fh_ != INVALID_ID){ + PIOc_closefile(fh_); + } + if(del_on_close_ && !name_.empty()){ + PIOc_deletefile(iosysid_, name_.c_str()); + } + } + private: + static const int INVALID_ID = -1; + /* Id of the I/O System associated with this file */ + int iosysid_; + /* Name of this file */ + const std::string name_; + int iotype_; + /* File handle/id of this file */ + int fh_; + bool is_in_def_mode_; + bool del_on_close_; + /* Local cache of file objects (data) in this file */ + std::map > dcache_; + + int create_file(int iosysid, const std::string name, int iotype){ + int ret = PIO_NOERR; + + ret = PIOc_createfile(iosysid, &fh_, &iotype, name.c_str(), PIO_CLOBBER); + if(ret != PIO_NOERR){ + fh_ = INVALID_ID; + std::string err_msg("PIOc_createfile("); + err_msg += "file = " + name + + ",iotype = " + std::to_string(iotype) + + ") failed, ret = " + std::to_string(ret) + "\n"; + + Util::GVars::logger->log(Util::Logging::LogLevel::ERROR, err_msg); + } + + return PIO_NOERR; + } + }; + + /* File dimension class */ + class SPIO_dim : public SPIO_valid_dim{ + public: + SPIO_dim(const std::string &name, PIO_Offset sz):name_(name), id_(INVALID_ID), + sz_(sz), fh_(INVALID_ID) {} + + /* To avoid collision with variable names, dim names need to be decorated */ + std::string name(void ) const override { return name_to_dname(name_); } + + /* Convert between decorated (used in app) and internal (used in file) dimension names */ + static std::string name_to_dname(const std::string &name) { return get_dname_prefix() + name; } + static std::string dname_to_name(const std::string &dname){ + /* FIXME: Use static and move it to cpp */ + const std::string dname_prefix(get_dname_prefix()); + + std::string name = dname; + if(name.find(dname_prefix) == 0){ + name.erase(0, dname_prefix.size()); + } + + return name; + } + int id(void ) const override { return id_; } + PIO_Offset size(void ) const { return sz_; } + + /* Define the file dimension */ + int def(SPIO_file &fh) override { + int ret = PIO_NOERR; + + fh_ = fh.id(); + fname_ = fh.name(); + Util::GVars::logger->log(Util::Logging::LogLevel::VERBOSE, "Defining dimension : " + name_ + + " in file " + fname_ + "\n"); + ret = PIOc_def_dim(fh_, name_.c_str(), sz_, &id_); + if(ret != PIO_NOERR){ + std::string err_msg("PIOc_def_dim("); + err_msg += "file = " + fname_ + + "dim name = " + name_ + + ", sz = " + std::to_string(sz_) + + ") failed, ret = " + std::to_string(ret) + "\n"; + + Util::GVars::logger->log(Util::Logging::LogLevel::ERROR, err_msg); + return ret; + } + + return PIO_NOERR; + } + + /* Since the dimension is already defined, nothing to be done here */ + int put(void) override { + return PIO_NOERR; + } + + /* Read the dimension and verify */ + int get_and_verify(void ) const override { + int ret = PIO_NOERR; + + Util::GVars::logger->log(Util::Logging::LogLevel::VERBOSE, "Verifying dimension info : " + name_ + + " in file " + fname_ + "\n"); + int gid = INVALID_ID; + ret = PIOc_inq_dimid(fh_, name_.c_str(), &gid); + if(ret != PIO_NOERR){ + std::string err_msg("PIOc_inq_dimid("); + err_msg += "file = " + fname_ + + "dim name = " + name_ + + ", sz = " + std::to_string(sz_) + + ") failed, ret = " + std::to_string(ret) + "\n"; + + Util::GVars::logger->log(Util::Logging::LogLevel::ERROR, err_msg); + return ret; + } + + if(gid != id_){ + std::string err_msg("Invalid dimension id retrieved from file"); + err_msg += "(file = " + fname_ + + "dim name = " + name_ + + ", sz = " + std::to_string(sz_) + + ", expected id = " + std::to_string(id_) + + ", id from file = " + std::to_string(gid) + + ") failed, ret = " + std::to_string(ret) + "\n"; + + Util::GVars::logger->log(Util::Logging::LogLevel::ERROR, err_msg); + return ret; + } + + char dname[PIO_MAX_NAME + 1]; + PIO_Offset gsz = 0; + ret = PIOc_inq_dim(fh_, id_, dname, &gsz); + if(ret != PIO_NOERR){ + std::string err_msg("PIOc_inq_dim("); + err_msg += "file = " + fname_ + + "dim name = " + name_ + + ", sz = " + std::to_string(sz_) + + ") failed, ret = " + std::to_string(ret) + "\n"; + + Util::GVars::logger->log(Util::Logging::LogLevel::ERROR, err_msg); + return ret; + } + + if((std::string(dname) != name_) || (gsz != sz_)){ + std::string err_msg("Invalid dimension name/size retrieved from file"); + err_msg += "(file = " + fname_ + + "dim name = " + name_ + + ", expected sz = " + std::to_string(sz_) + + ", sz from file = " + std::to_string(gsz) + + ", expected name = " + name_ + + ", name from file = " + std::string(dname) + + ")\n"; + + Util::GVars::logger->log(Util::Logging::LogLevel::ERROR, err_msg); + return ret; + } + + return PIO_NOERR; + } + + private: + static const int INVALID_ID = -1; + + /* Name of the dimension. This name is not decorated, however all queries to this + * class for the name returns the decorated name (get_dname_prefix() + name_) + */ + const std::string name_; + int id_; + /* Size of the dimension */ + PIO_Offset sz_; + /* Cached file handle associated with this dimension */ + int fh_; + /* Cached name of the file associated with this dimension */ + std::string fname_; + + static inline std::string get_dname_prefix(void ) { return std::string("__SPIO_dim__"); } + }; + + /* An attribute class. Only char/int/float/duoble supported for now */ + /* FIXME: Use a variant instead */ + class SPIO_att : public SPIO_file_obj{ + public: + SPIO_att(const std::string &name, const std::string &val) : + name_(name), id_(INVALID_ID), fid_(INVALID_ID), vid_(INVALID_ID), + pio_type_(PIO_CHAR), sz_(val.size()), sval_(val){} + SPIO_att(const std::string &name, int val) : + name_(name), id_(INVALID_ID), fid_(INVALID_ID), vid_(INVALID_ID), + pio_type_(PIO_INT), sz_(1), ival_(val){} + SPIO_att(const std::string &name, float val) : + name_(name), id_(INVALID_ID), fid_(INVALID_ID), vid_(INVALID_ID), + pio_type_(PIO_FLOAT), sz_(1), fval_(val){} + SPIO_att(const std::string &name, double val) : + name_(name), id_(INVALID_ID), fid_(INVALID_ID), vid_(INVALID_ID), + pio_type_(PIO_DOUBLE), sz_(1), dval_(val){} + + std::string name(void ) const override { return name_; } + int id(void ) const override { return id_; } + int type(void ) const { return pio_type_; } + + /* Get the attribute value */ + template + T val(void) const{ + switch(pio_type_){ + case PIO_CHAR: return sval_; + case PIO_INT: return ival_; + case PIO_FLOAT: return fval_; + case PIO_DOUBLE : return dval_; + } + throw std::runtime_error("Invalid attribute type, getting attribute val failed"); + } + + /* Define a global file attribute */ + int def(SPIO_file &f) override{ + fid_ = f.id(); + fname_ = f.name(); + vid_ = PIO_GLOBAL; + vname_ = "PIO_GLOBAL"; + + Util::GVars::logger->log(Util::Logging::LogLevel::VERBOSE, "Defining global attribute : " + name_ + + " in file " + fname_ + "\n"); + return put(fid_, vid_); + } + + /* Define a variable attribute */ + int def(SPIO_file &f, SPIO_valid_var &v){ + fid_ = f.id(); + fname_ = f.name(); + vid_ = v.id(); + vname_ = v.name(); + + Util::GVars::logger->log(Util::Logging::LogLevel::VERBOSE, "Defining variable attribute : " + name_ + + " for variable " + vname_ + " in file " + fname_ + "\n"); + return put(fid_, vid_); + } + + /* Attribute is defined already, nothing to do here */ + int put(void ) override{ + return PIO_NOERR; + } + + /* Read and verify the stored attributes */ + int get_and_verify(void ) const override{ + int ret = PIO_NOERR; + assert((fid_ != INVALID_ID) && (vid_ != INVALID_ID)); + + Util::GVars::logger->log(Util::Logging::LogLevel::VERBOSE, "Verifying attribute : " + name_ + + " in file " + fname_ + "\n"); + switch(pio_type_){ + case PIO_CHAR: + { + char gval[PIO_MAX_NAME + 1] = "\0"; + ret = PIOc_get_att_text(fid_, vid_, name_.c_str(), gval); + if(ret == PIO_NOERR){ + if(sval_ != gval){ + std::string err_msg = "PIOc_get_att_text(file=" + fname_ + ", vname = " + vname_ + ", att_name = " + name_ + ") failed, expected = " + sval_ + ", read = " + std::string(gval) + "\n"; + Util::GVars::logger->log(Util::Logging::LogLevel::ERROR, err_msg); + return PIO_EINTERNAL; + } + } + break; + } + case PIO_INT: + { + int gval = 0; + ret = PIOc_get_att(fid_, vid_, name_.c_str(), &gval); + if(ret == PIO_NOERR){ + if(ival_ != gval){ + std::string err_msg = "PIOc_get_att(file=" + fname_ + ", vname = " + vname_ + ", att_name = " + name_ + ") failed, expected = " + std::to_string(ival_) + ", read = " + std::to_string(gval) + "\n"; + Util::GVars::logger->log(Util::Logging::LogLevel::ERROR, err_msg); + return PIO_EINTERNAL; + } + } + break; + } + case PIO_FLOAT: + { + float gval = 0.0; + ret = PIOc_get_att(fid_, vid_, name_.c_str(), &gval); + if(ret == PIO_NOERR){ + if(fval_ != gval){ + std::string err_msg = "PIOc_get_att(file=" + fname_ + ", vname = " + vname_ + ", att_name = " + name_ + ") failed, expected = " + std::to_string(fval_) + ", read = " + std::to_string(gval) + "\n"; + Util::GVars::logger->log(Util::Logging::LogLevel::ERROR, err_msg); + return PIO_EINTERNAL; + } + } + break; + } + case PIO_DOUBLE: + { + double gval = 0.0; + ret = PIOc_get_att(fid_, vid_, name_.c_str(), &gval); + if(ret == PIO_NOERR){ + if(dval_ != gval){ + std::string err_msg = "PIOc_get_att(file=" + fname_ + ", vname = " + vname_ + ", att_name = " + name_ + ") failed, expected = " + std::to_string(dval_) + ", read = " + std::to_string(gval) + "\n"; + Util::GVars::logger->log(Util::Logging::LogLevel::ERROR, err_msg); + return PIO_EINTERNAL; + } + } + break; + } + default: + throw std::runtime_error("PIOc_get_att() failed, unsupported type"); + } + if(ret != PIO_NOERR){ + std::string err_msg = "PIOc_get_att() failed : "; + err_msg += ", fname = " + fname_ + + ", vname = " + vname_ + + ", att_name = " + name_ + + ", pio_type = " + std::to_string(pio_type_) + "\n"; + + Util::GVars::logger->log(Util::Logging::LogLevel::ERROR, err_msg); + return ret; + } + return PIO_NOERR; + } + + private: + static const int INVALID_ID = -2; + + /* Name of the attribute */ + const std::string name_; + /* Name of file/variable associated with the attribute */ + std::string fname_; + std::string vname_; + + /* Id of the attribute */ + int id_; + /* Id of file/variable associated with the attribute */ + int fid_; + int vid_; + int pio_type_; + int sz_; + + std::string sval_; + /* Union is probably an overkill here */ + int ival_; + float fval_; + double dval_; + + int put(int fh, int vid){ + int ret = PIO_NOERR; + switch(pio_type_){ + case PIO_CHAR: + ret = PIOc_put_att_text(fh, vid, name_.c_str(), sz_, static_cast(val_ptr())); break; + default: + ret = PIOc_put_att(fh, vid, name_.c_str(), pio_type_, sz_, val_ptr()); + } + if(ret != PIO_NOERR){ + std::string err_msg = "PIOc_put_att() failed : "; + err_msg += ", fname = " + fname_ + + ", vname = " + vname_ + + ". name = " + name_ + + ", pio_type = " + std::to_string(pio_type_) + "\n"; + + Util::GVars::logger->log(Util::Logging::LogLevel::ERROR, err_msg); + return ret; + } + + return PIO_NOERR; + } + + /* Get a pointer to the stored attribute value */ + const void *val_ptr(void){ + switch(pio_type_){ + case PIO_CHAR: return static_cast(sval_.c_str()); + case PIO_INT: return static_cast(&ival_); + case PIO_FLOAT: return static_cast(&fval_); + case PIO_DOUBLE : return static_cast(&dval_); + default: + throw std::runtime_error("Invalid attribute type, getting attribute val failed"); + } + } + }; + + /* Generic variable class */ + template + class SPIO_var : public SPIO_valid_var{ + public: + SPIO_var(const std::string &name, + const std::vector &dims, + const std::vector &atts, + const std::vector &val) : + name_(name), id_(INVALID_ID), fid_(INVALID_ID), dims_(dims), gsz_(0), + atts_(atts), val_(val) {} + + SPIO_var(const std::string &name, + const std::vector &dims, + const std::vector &atts, + std::function &val_generator) : + name_(name), id_(INVALID_ID), fid_(INVALID_ID), dims_(dims), gsz_(0), + atts_(atts), val_gen_(val_generator) {} + + std::string name(void ) const override { return name_; } + int id(void ) const override { return id_; } + + /* Define a variable */ + int def(SPIO_file &f) override{ + int ret = PIO_NOERR; + + init_file_info(f); + + Util::GVars::logger->log(Util::Logging::LogLevel::VERBOSE, "Defining variable : " + name_ + + " in file " + fname_ + "\n"); + ret = PIOc_def_var(fid_, name_.c_str(), + Type_Traits::cxx_to_pio_type::pio_type, dim_ids_.size(), + dim_ids_.data(), &id_); + if(ret != PIO_NOERR){ + std::string err_msg("PIOc_def_var"); + err_msg += "file = " + fname_ + + ", name = " + name_ + + ", dims = " + Util::String::vec_to_string(dims_) + + ", dim sizes = " + Util::String::vec_to_string(dim_sz_) + + ") failed, ret = " + std::to_string(ret) + "\n"; + + Util::GVars::logger->log(Util::Logging::LogLevel::ERROR, err_msg); + return ret; + } + + for(std::vector::iterator iter = atts_.begin(); + iter != atts_.end(); ++iter){ + ret = iter->def(f, *this); + if(ret != PIO_NOERR){ + Util::GVars::logger->log(Util::Logging::LogLevel::ERROR, + "Defining attribute for variable : " + name_ + " failed\n"); + return ret; + } + } + return PIO_NOERR; + } + + /* Write the variable data to file */ + int put(void ) override{ + int ret = PIO_NOERR; + + Util::GVars::logger->log(Util::Logging::LogLevel::VERBOSE, "Writing variable : " + name_ + + " in file " + fname_ + "\n"); + SPIO_var::init_val(); + + ret = PIOc_put_var(fid_, id_, val_.data()); + if(ret != PIO_NOERR){ + std::string err_msg("PIOc_put_var"); + err_msg += "file = " + fname_ + + ", name = " + name_ + + ", dims = " + Util::String::vec_to_string(dims_) + + ", dim sizes = " + Util::String::vec_to_string(dim_sz_) + + ") failed, ret = " + std::to_string(ret) + "\n"; + + Util::GVars::logger->log(Util::Logging::LogLevel::ERROR, err_msg); + return ret; + } + return PIO_NOERR; + } + + int get_and_verify(void ) const override{ + /* FIXME: Add code to read and verify the written data */ + /* + int ret = PIO_NOERR; + + switch(Type_Traits::cxx_to_pio_type::pio_type){ + case PIO_CHAR: + case PIO_INT: + case PIO_FLOAT: + case PIO_DOUBLE: + } + */ + + return PIO_NOERR; + } + + virtual ~SPIO_var(){} + + protected: + static const int INVALID_ID = -1; + + const std::string name_; + std::string fname_; + int id_; + int fid_; + /* Dimension names for the variable */ + std::vector dims_; + /* Dimension ids of the variable */ + std::vector dim_ids_; + /* Dimension sizes of the variable */ + std::vector dim_sz_; + /* Size of the variable */ + std::size_t gsz_; + /* Variable attributes */ + std::vector atts_; + + /* Variable value */ + std::vector val_; + /* Since strings are converted to character arrays before + * passing it on to SPIO apis we have a cval_[] buffer + * (that is only valid for char/string variables) + */ + std::vector cval_; + /* Fillvalue */ + std::vector fval_; + std::function val_gen_; + + virtual void init_val(void ){ + if(val_gen_){ + assert(val_.size() == 0); + std::generate_n(std::back_inserter(val_), gsz_, val_gen_); + } + } + + /* Initialize the file info - id/name/dims etc */ + void init_file_info(const SPIO_file &f) + { + assert(fid_ == INVALID_ID); + fid_ = f.id(); + fname_ = f.name(); + + if(dims_.size() == 0){ + gsz_ = 0; + return; + } + + gsz_ = 1; + std::for_each(dims_.cbegin(), dims_.cend(), + [&f, this](const std::string &dname){ + /* Note that we always use the decorated dimension name for internal caching + * - to avoid colissions with variable names (i.e., name_to_dname(DIM_NAME)) + */ + std::shared_ptr pdim = + std::dynamic_pointer_cast(f.get(SPIO_dim::name_to_dname(dname))); + assert(pdim); + + dim_ids_.push_back(pdim->id()); + + PIO_Offset dim_sz = pdim->size(); + dim_sz_.push_back(dim_sz); + gsz_ *= (dim_sz != PIO_UNLIMITED) ? dim_sz : 1; + }); + } + }; + + /* Specialized function to initialize var with strings */ + template<> + inline void SPIO_var::init_val(void ){ + if(val_gen_){ + assert(val_.size() == 0); + std::generate_n(std::back_inserter(val_), gsz_, val_gen_); + } + + /* Copy strings in val_ to a char vector, cval_ */ + std::for_each(val_.cbegin(), val_.cend(), + [this](const std::string &str){ + std::copy(str.cbegin(), str.cend(), std::back_inserter(cval_)); + }); + } + + /* Variable class for variables written out with starts/counts */ + template + class SPIO_cs_var : public SPIO_var{ + public: + SPIO_cs_var(const std::string &name, + const std::vector &dims, + const std::vector &atts, + const std::vector > &starts, + const std::vector > &counts, + const std::vector &val): + SPIO_var(name, dims, atts, val), starts_(starts), counts_(counts) {} + SPIO_cs_var(const std::string &name, + const std::vector &dims, + const std::vector &atts, + const std::vector > &starts, + const std::vector > &counts, + std::function &val_generator) : + SPIO_var(name, dims, atts, val_generator), + starts_(starts), counts_(counts) {} + + /* We use the starts/counts to write the variable here */ + int put(void ) override{ + int ret = PIO_NOERR; + + /* Initialize the variable value */ + SPIO_var::init_val(); + + Util::GVars::logger->log(Util::Logging::LogLevel::VERBOSE, + "Writing variable (starts+counts) : " + SPIO_var::name_ + + " in file " + SPIO_var::fname_ + "\n"); + assert(starts_.size() == counts_.size()); + PIO_Offset val_idx = 0; + for(std::vector >::const_iterator + citer1 = starts_.cbegin(), citer2 = counts_.cbegin(); + (citer1 != starts_.cend()) && (citer2 != counts_.cend()); ++citer1, ++citer2){ + + assert((*citer1).size() == (*citer2).size()); + ret = put(SPIO_var::fid_, SPIO_var::id_, *citer1, *citer2, val_idx); + if(ret != PIO_NOERR){ + std::string err_msg("PIOc_put_var[a/s]"); + err_msg += "file = " + SPIO_var::fname_ + + ", name = " + SPIO_var::name_ + + ", dims = " + Util::String::vec_to_string(SPIO_var::dims_) + + ", dim sizes = " + Util::String::vec_to_string(SPIO_var::dim_sz_) + + ") failed, ret = " + std::to_string(ret) + "\n"; + + Util::GVars::logger->log(Util::Logging::LogLevel::ERROR, err_msg); + return ret; + } + + /* Update the index/offset, val_idx, in the val_ buffer based on the count[] + * written out + */ + PIO_Offset nvals = std::accumulate((*citer2).cbegin(), (*citer2).cend(), + 1, std::multiplies()); + val_idx += nvals; + if(SPIO_var::cval_.size() == 0){ + if(static_cast(val_idx) >= SPIO_var::val_.size()) { val_idx = 0; } + } + else{ + if(static_cast(val_idx) >= SPIO_var::cval_.size()) { val_idx = 0; } + } + } + return PIO_NOERR; + } + + private: + const std::vector > starts_; + const std::vector > counts_; + + int put(int fid, int vid, const std::vector &start, + const std::vector &count, PIO_Offset val_idx); + }; + + /* Specialized put() functions for different types of variable data */ + template<> + inline int SPIO_cs_var::put(int fid, int vid, const std::vector &start, + const std::vector &count, + PIO_Offset val_idx) + { + return PIOc_put_vara_text(fid, vid, start.data(), count.data(), &cval_[val_idx]); + } + + template<> + inline int SPIO_cs_var::put(int fid, int vid, const std::vector &start, + const std::vector &count, + PIO_Offset val_idx) + { + return PIOc_put_vars_int(fid, vid, start.data(), count.data(), NULL, &val_[val_idx]); + } + + template<> + inline int SPIO_cs_var::put(int fid, int vid, const std::vector &start, + const std::vector &count, + PIO_Offset val_idx) + { + return PIOc_put_vars_float(fid, vid, start.data(), count.data(), NULL, &val_[val_idx]); + } + + template<> + inline int SPIO_cs_var::put(int fid, int vid, const std::vector &start, + const std::vector &count, + PIO_Offset val_idx) + { + return PIOc_put_vars_double(fid, vid, start.data(), count.data(), NULL, &val_[val_idx]); + } + + /* Variable with unlimited dimension */ + template + class SPIO_unlimited_var : public SPIO_var{ + public: + SPIO_unlimited_var(const std::string &name, + const std::vector &dims, + const std::vector &atts, + std::shared_ptr pdecomp, + int nframes, + const std::vector &val): + SPIO_var(name, dims, atts, val), pdecomp_(pdecomp), nframes_(nframes){} + + SPIO_unlimited_var(const std::string &name, + const std::vector &dims, + const std::vector &atts, + std::shared_ptr &pdecomp, + int nframes, + std::function &val_generator): + SPIO_var(name, dims, atts, val_generator), pdecomp_(pdecomp), nframes_(nframes){} + + /* Write/Put using associated I/O decomp */ + int put(void ) override{ + int ret = PIO_NOERR; + + Util::GVars::logger->log(Util::Logging::LogLevel::VERBOSE, + "Writing variable (unlimited dim) : " + SPIO_var::name_ + + " in file " + SPIO_var::fname_ + "\n"); + SPIO_var::init_val(); + + /* Data is written out one frame (unlimited dim) at a time */ + for(int iframe=0; iframe < nframes_; iframe++){ + /* Set the frame being written out */ + ret = PIOc_setframe(SPIO_var::fid_, SPIO_var::id_, iframe); + if(ret != PIO_NOERR){ + std::string err_msg("PIOc_setframe"); + err_msg += "file = " + SPIO_var::fname_ + + ", name = " + SPIO_var::name_ + + ", dims = " + Util::String::vec_to_string(SPIO_var::dims_) + + ", dim sizes = " + Util::String::vec_to_string(SPIO_var::dim_sz_) + + ", frame = " + std::to_string(iframe) + + ") failed, ret = " + std::to_string(ret) + "\n"; + + Util::GVars::logger->log(Util::Logging::LogLevel::ERROR, err_msg); + return ret; + } + + ret = PIOc_write_darray(SPIO_var::fid_, SPIO_var::id_, + pdecomp_->id(), SPIO_var::val_.size(), SPIO_var::val_.data(), + (SPIO_var::fval_.size() != 0) ? SPIO_var::fval_.data() : NULL); + if(ret != PIO_NOERR){ + std::string err_msg("PIOc_write_darray"); + err_msg += "file = " + SPIO_var::fname_ + + ", name = " + SPIO_var::name_ + + ", dims = " + Util::String::vec_to_string(SPIO_var::dims_) + + ", dim sizes = " + Util::String::vec_to_string(SPIO_var::dim_sz_) + + ", frame = " + std::to_string(iframe) + + ") failed, ret = " + std::to_string(ret) + "\n"; + + Util::GVars::logger->log(Util::Logging::LogLevel::ERROR, err_msg); + return ret; + } + } + + return PIO_NOERR; + } + + private: + /* I/O decomposition associated with this variable */ + std::shared_ptr pdecomp_; + /* Number of frames in this variable */ + int nframes_; + }; + +} // namespace Util + +#endif // __E3SM_FGI_DATA_HPP__ diff --git a/examples/cxx/e3sm_fgi_utils.hpp b/examples/cxx/e3sm_fgi_utils.hpp new file mode 100644 index 0000000000..55986b7ca4 --- /dev/null +++ b/examples/cxx/e3sm_fgi_utils.hpp @@ -0,0 +1,265 @@ +#ifndef __E3SM_UTILS_FGI_HPP__ +#define __E3SM_UTILS_FGI_HPP__ + +#include +#ifdef SPIO_ENABLE_GPTL_TIMING +#include +#endif +#include "pio.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace E3SM_FGI{ + enum class Case_Type{ + E3SM_F_CASE, + E3SM_G_CASE, + E3SM_I_CASE + }; +} + +/* FIXME: Move logging related classes to a util */ +namespace Util{ + namespace String{ + /* Util to convert a vactor of C++ types to a single string */ + template + inline std::string vec_to_string(const std::vector &vec) + { + std::string str("["); + str += std::accumulate(vec.cbegin(), vec.cend(), std::string(""), + [](std::string str, const T &t){ + return std::to_string(t) + ", " + str; + }); + str += "]"; + return str; + } + + /* Util to convert a vactor of C++ strings to a single string */ + template<> + inline std::string vec_to_string(const std::vector &vec) + { + std::string str("["); + str += std::accumulate(vec.cbegin(), vec.cend(), std::string(""), + [](std::string res, const std::string &str){ + return "\"" + str + "\", " + res; + }); + str += "]"; + return str; + } + + /* toupper() for a string */ + static inline std::string toupper(const std::string &str) + { + std::string res = str; + std::transform(str.cbegin(), str.cend(), res.begin(), + [](unsigned char c) { return std::toupper(c); }); + return res; + } + } + + namespace Logging{ + /* Different log levels supported */ + enum class LogLevel{ STATUS, VERBOSE, WARNING, ERROR }; + static inline std::string llevel2str(LogLevel lvl){ + std::string str; + switch(lvl){ + case LogLevel::STATUS : str = "STATUS"; break; + case LogLevel::VERBOSE : str = "VERBOSE"; break; + case LogLevel::WARNING : str = "WARNING"; break; + case LogLevel::ERROR : str = "ERROR"; break; + } + return str; + } + + /* A message logger : To log/print to stdout, use "STDOUT" for the logger name */ + class Logger{ + public: + static std::shared_ptr get_logger(MPI_Comm comm, const std::string &log_name){ + static std::shared_ptr logger_instance; + return logger_instance ? logger_instance : std::shared_ptr(is_std_logname(log_name) ? new Logger(comm) : new Logger(comm, log_name)); + } + + /* Log message (at level lvl) */ + void log(LogLevel lvl, const std::string &str){ + if(!need_to_log(lvl)) return; + + if((log_rank_ == LOG_ALL_RANKS) || (rank_ == log_rank_)){ + if(log_fname_.empty()){ + std::cout << llevel2str(lvl).c_str() << ":" << str.c_str() << std::flush; + } + else{ + if(!log_fstr_.is_open()){ + log_fstr_.open(log_fname_); + } + log_fstr_ << llevel2str(lvl).c_str() << ":" << str.c_str(); + } + } + } + + /* Set the MPI process rank - that logs the message */ + Logger &set_log_rank(int rank){ + log_rank_ = rank; + if(!log_fname_.empty()){ + log_fname_ = get_log_fname(); + } + return *this; + } + + /* Set log level */ + Logger &set_log_level(LogLevel lvl) { log_lvl_ = lvl; return *this; } + + ~Logger(){ + if(log_fstr_.is_open()){ + log_fstr_.close(); + } + } + private: + Logger(MPI_Comm comm) : rank_(0), log_rank_(LOG_ALL_RANKS){ + int ret = MPI_Comm_rank(comm, &rank_); assert(ret == MPI_SUCCESS); + } + + Logger(MPI_Comm comm, const std::string &log_fname) : rank_(0), log_rank_(LOG_ALL_RANKS), log_fname_prefix_(log_fname){ + int ret = MPI_Comm_rank(comm, &rank_); assert(ret == MPI_SUCCESS); + log_fname_ = get_log_fname(); + } + + /* Check if the user wants to log to stdout */ + static bool is_std_logname(const std::string &log_name) { return (Util::String::toupper(log_name) == "STDOUT"); } + std::string get_log_fname(void ) const { return log_fname_prefix_ + "_" + std::to_string(log_rank_) + ".log"; } + + bool need_to_log(LogLevel msg_lvl) const { + switch(log_lvl_){ + case LogLevel::VERBOSE : return true; + case LogLevel::STATUS : return (msg_lvl != LogLevel::VERBOSE) ? true : false; + case LogLevel::WARNING : return ((msg_lvl == LogLevel::ERROR) || (msg_lvl == LogLevel::WARNING)) ? true : false; + case LogLevel::ERROR : return (msg_lvl == LogLevel::ERROR) ? true : false; + } + return false; + } + + const int LOG_ALL_RANKS = -1; + int rank_; + int log_rank_; + std::string log_fname_prefix_; + std::string log_fname_; + std::ofstream log_fstr_; + LogLevel log_lvl_; + }; + } // namespace Logging +} // namespace Util + +namespace Util{ + /* Global variable declarations (see e3sm_fgi.cpp) and + * functions to modify them + */ + namespace GVars{ + /* Available I/O types - for parsing user args*/ + extern std::unordered_map iotypes; + /* Available I/O rearrangers - for parsing user args */ + extern std::unordered_map rearrs; + /* E3SM pseudo cases to test - for parsing user args */ + extern std::unordered_map cases; + /* Available log levels - for parsing user args */ + extern std::unordered_map llevels; + + /* Convert user option to string using the provided "option map", opt_map */ + template + std::string opt_map_to_str(const std::unordered_map &opt_map) + { + std::string str("["); + str += std::accumulate(opt_map.cbegin(), opt_map.cend(), std::string(""), + [](std::string str, const std::pair &t){ + return t.first + ", " + str; + }); + str += "]"; + return str; + } + + /* Convert option string to option type using provided "option map" */ + template + std::string opt_type_to_str(T opt_type, + const std::unordered_map &opt_map) + { + typename std::unordered_map::const_iterator iter = + std::find_if(opt_map.cbegin(), opt_map.cend(), + [opt_type](const std::pair &a) { return a.second == opt_type; }); + if(iter != opt_map.cend()){ + return iter->first; + } + else{ + throw std::runtime_error("Unable to find key in option map"); + } + } + + /* Copy the available options, the option names, to the destination */ + template + void copy_opt_map(const std::unordered_map &opt_map, + InsertIter dest) + { + std::transform(opt_map.cbegin(), opt_map.cend(), dest, + [](const std::pair &a) { return a.second; }); + } + + /* Helper utils for parsing user options, converting between types and names */ + static inline std::string iotypes2str(void ){ return opt_map_to_str(iotypes); } + static inline std::string rearrs2str(void ){ return opt_map_to_str(rearrs); } + static inline std::string cases2str(void ){ return opt_map_to_str(cases); } + static inline std::string llevels2str(void ){ return opt_map_to_str(llevels); } + + static inline std::string iotype2str(int iotype){ return opt_type_to_str(iotype, iotypes); } + static inline int str2iotype(const std::string &iotype_str){ return iotypes.at(iotype_str); } + static inline std::string rearr2str(int rearr){ return opt_type_to_str(rearr, rearrs); } + static inline int str2rearr(const std::string &rearr_str){ return rearrs.at(rearr_str); } + static inline std::string case2str(E3SM_FGI::Case_Type c){ return opt_type_to_str(c, cases); } + static inline E3SM_FGI::Case_Type str2case(const std::string &case_str){ return cases.at(case_str); } + static inline std::string llevel2str(Util::Logging::LogLevel llevel){ return opt_type_to_str(llevel, llevels); } + static inline Util::Logging::LogLevel str2llevel(const std::string &llevel_str){ return llevels.at(llevel_str); } + + /* Decl for the global logger */ + extern std::shared_ptr logger; + } // namespace GVars +} //namespace Util + +namespace Util{ + /* Run function f for each pair from the input iterators */ + template + void zip_for_each(InputIterator1 iter1_begin, InputIterator1 iter1_end, InputIterator2 iter2_begin, InputIterator2 iter2_end, Func f) + { + for(InputIterator1 iter1 = iter1_begin; iter1 != iter1_end; ++iter1){ + for(InputIterator2 iter2 = iter2_begin; iter2 != iter2_end; ++iter2){ + f(*iter1, *iter2); + } + } + } + + /* Util to check return value from the lib */ + static inline int check_spio_err(int ret, const std::string &err_msg, const char *fname, int line_num) + { + if(ret != PIO_NOERR){ + Util::GVars::logger->log(Util::Logging::LogLevel::ERROR, + err_msg + ", file = " + std::string(fname) + ", line no : " + std::to_string(line_num)); + exit(-1); + } + + return ret; + } +} // namespace Util + +namespace E3SM_FGI{ + /* Decls of the pseudo E3SM case functions */ + int test_e3sm_fcase(MPI_Comm comm, const std::vector &iotypes, + const std::vector &rearrs, int nioprocs); + int test_e3sm_gcase(MPI_Comm comm, const std::vector &iotypes, + const std::vector &rearrs, int nioprocs); + int test_e3sm_icase(MPI_Comm comm, const std::vector &iotypes, + const std::vector &rearrs, int nioprocs); +} +#endif // __E3SM_UTILS_FGI_HPP__ diff --git a/examples/cxx/e3sm_g.cpp b/examples/cxx/e3sm_g.cpp new file mode 100644 index 0000000000..fc085c42f5 --- /dev/null +++ b/examples/cxx/e3sm_g.cpp @@ -0,0 +1,204 @@ +#include "e3sm_fgi_utils.hpp" +#include "e3sm_fgi_data.hpp" + +/* Get unique output file name */ +static inline std::string get_gcase_test_fname(const std::string &case_type, int iotype, int rearr) +{ + const std::string FNAME_PREFIX = "spio_e3sm_fgi_gcase"; + const std::string FNAME_SUFFIX = ".nc"; + const std::string SEP = "_"; + + return FNAME_PREFIX + + SEP + Util::GVars::iotype2str(iotype) + + SEP + Util::GVars::rearr2str(rearr) + + SEP + case_type + + FNAME_SUFFIX; +} + +/* Test pseudo G case */ +static int test_gcase(MPI_Comm comm, int iosysid, const std::string &fname, int iotype, + PIO_Offset ncells, PIO_Offset nedges, PIO_Offset nvertices, PIO_Offset nvertlevels, + int nframes) +{ + int comm_rank = 0, comm_sz = 0; + int ret = PIO_NOERR; + + + ret = MPI_Comm_rank(comm, &comm_rank); assert(ret == MPI_SUCCESS); + ret = MPI_Comm_size(comm, &comm_sz); assert(ret == MPI_SUCCESS); + + const PIO_Offset NEDGES_PER_PROC = std::max(static_cast(nedges / comm_sz), static_cast(1)); + const int STRLEN = 64; + const bool is_last_proc = (comm_rank == comm_sz - 1) ? true : false; + + PIO_Offset idx = 0; + std::function range_zero_to_inf = [idx] (void ) mutable { return static_cast(idx++); }; + + double dlev = 0; + std::function gen_levels = [dlev] (void ) mutable { double d = dlev; dlev += 0.2; return d; }; + + /* 1D decomp : the edges are evenly partitioned across all MPI processes */ + /* FIXME: This is not how MPAS paritions cells */ + std::function decomp_1d_gen_nedges = + [comm_rank, comm_sz, nedges, NEDGES_PER_PROC, idx](void ) mutable { + static const PIO_Offset INIT_IDX = 0; + const bool is_last_rank = (comm_rank == comm_sz - 1) ? true : false; + + PIO_Offset val = comm_rank * NEDGES_PER_PROC + idx++ + 1; + + if(!is_last_rank){ + if(idx >= NEDGES_PER_PROC) { idx = INIT_IDX; } + } + else{ + if(val >= nedges) { idx = INIT_IDX; } + } + + return val; + }; + + E3SM_FGI::SPIO_decomp decomp_1d("decomp_1d_nedges", iosysid, PIO_DOUBLE, + std::vector({static_cast(nedges)}), static_cast(NEDGES_PER_PROC), decomp_1d_gen_nedges); + + /* 2D decomp : the edges are evenly partitioned across all MPI processes */ + /* FIXME: This is not how MPAS paritions cells */ + std::function decomp_2d_gen_nedges = + [comm_rank, is_last_proc, nedges, NEDGES_PER_PROC, nvertlevels, idx](void ) mutable { + static const PIO_Offset INIT_IDX = 0; + + PIO_Offset val = comm_rank * (NEDGES_PER_PROC * nvertlevels) + idx++ + 1; + + if(!is_last_proc){ + if(idx >= (comm_rank + 1) * NEDGES_PER_PROC * nvertlevels) { idx = INIT_IDX; } + } + else{ + if(idx >= (nedges - comm_rank * NEDGES_PER_PROC) * nvertlevels) { idx = INIT_IDX; } + } + + return val; + }; + + std::function decomp_2d_edges_val_gen = + [decomp_2d_gen_nedges](void ){ + return static_cast(decomp_2d_gen_nedges()); + }; + + int lsz = static_cast(nvertlevels * + ((!is_last_proc) ? NEDGES_PER_PROC : (nedges - comm_rank * NEDGES_PER_PROC))); + E3SM_FGI::SPIO_decomp decomp_2d_edges("decomp_2d_nedges_nvertlevels", iosysid, PIO_DOUBLE, + std::vector({static_cast(nedges), static_cast(nvertlevels)}), + lsz, decomp_2d_gen_nedges); + std::shared_ptr pdecomp_2d_edges = + std::make_shared(std::move(decomp_2d_edges)); + + E3SM_FGI::SPIO_file fh(iosysid, fname, iotype, false); + + /* Define var/dim/atts */ + ret = fh.def({ + /* Global Attributes */ + std::make_shared("title", "MPAS History file information"), + std::make_shared("source", "E3SM Atmosphere Model"), + /* Dimensions */ + std::make_shared("Time", PIO_UNLIMITED), + std::make_shared("nCells", ncells), + std::make_shared("nEdges", nedges), + std::make_shared("nVertices", nvertices), + std::make_shared("nVertLevels", nvertlevels), + std::make_shared("nVertLevelsP1", nvertlevels), + std::make_shared("StrLen", STRLEN), + /* Variables */ + std::make_shared >("normalVerlocity", std::vector({"Time", "nEdges", "nVertLevels"}), std::vector({}), pdecomp_2d_edges, nframes, decomp_2d_edges_val_gen) + }); + if(ret != PIO_NOERR){ + Util::GVars::logger->log(Util::Logging::LogLevel::ERROR, "Defining file objects failed"); + return ret; + } + + /* Write var/atts */ + ret = fh.put(); + if(ret != PIO_NOERR){ + Util::GVars::logger->log(Util::Logging::LogLevel::ERROR, "Putting file objects failed"); + return ret; + } + + return ret; +} + +/* Test small G case */ +static int test_gcase_small(MPI_Comm comm, int iosysid, const std::string &fname, int iotype) +{ + int comm_rank = 0, comm_sz = 0; + int ret = PIO_NOERR; + + Util::GVars::logger->log(Util::Logging::LogLevel::STATUS, "Testing small G Case , iotype = " + Util::GVars::iotype2str(iotype) + "\n"); + + ret = MPI_Comm_rank(comm, &comm_rank); assert(ret == MPI_SUCCESS); + ret = MPI_Comm_size(comm, &comm_sz); assert(ret == MPI_SUCCESS); + + const PIO_Offset NCELLS = 32; + const PIO_Offset NEDGES = 32; + const PIO_Offset NVERTICES = 10; + const PIO_Offset NVERTLEVELS = 11; + const int NFRAMES = 2; + + return test_gcase(comm, iosysid, fname, iotype, NCELLS, NEDGES, NVERTICES, NVERTLEVELS, NFRAMES); + +} + +/* Test large G case : this can be too big to run on a single compute node */ +static int test_gcase_large(MPI_Comm comm, int iosysid, const std::string &fname, int iotype) +{ + int comm_rank = 0, comm_sz = 0; + int ret = PIO_NOERR; + + Util::GVars::logger->log(Util::Logging::LogLevel::STATUS, "Testing large G Case , iotype = " + Util::GVars::iotype2str(iotype) + "\n"); + + ret = MPI_Comm_rank(comm, &comm_rank); assert(ret == MPI_SUCCESS); + ret = MPI_Comm_size(comm, &comm_sz); assert(ret == MPI_SUCCESS); + + const PIO_Offset NCELLS = 3693225; + const PIO_Offset NEDGES = 11135652; + const PIO_Offset NVERTICES = 7441216; + const PIO_Offset NVERTLEVELS = 81; + const int NFRAMES = 2; + + return test_gcase(comm, iosysid, fname, iotype, NCELLS, NEDGES, NVERTICES, NVERTLEVELS, NFRAMES); + +} + +int E3SM_FGI::test_e3sm_gcase(MPI_Comm comm, const std::vector &iotypes, + const std::vector &rearrs, int nioprocs) +{ + int comm_rank, comm_sz, ret; + ret = MPI_Comm_rank(comm, &comm_rank); assert(ret == MPI_SUCCESS); + ret = MPI_Comm_size(comm, &comm_sz); assert(ret == MPI_SUCCESS); + + assert(nioprocs <= comm_sz); + + int io_stride = comm_sz / nioprocs; + + /* Run G case for all combinations of I/O types and I/O rearrangers */ + Util::zip_for_each(iotypes.cbegin(), iotypes.cend(), rearrs.cbegin(), rearrs.cend(), + [comm, nioprocs, io_stride](int iotype, int rearr){ + const int IOSYS_START_PROC = 0; + const int SPIO_IOSYSID_INVALID = -2; + int ret = PIO_NOERR; + + Util::GVars::logger->log(Util::Logging::LogLevel::STATUS, "Testing G Case, iotype = " + Util::GVars::iotype2str(iotype) + ", rearr = " + Util::GVars::rearr2str(rearr) + "\n"); + int iosysid = SPIO_IOSYSID_INVALID; + ret = PIOc_Init_Intracomm(comm, nioprocs, io_stride, IOSYS_START_PROC, + rearr, &iosysid); + Util::check_spio_err(ret, "PIOc_Init_Intracomm failed", __FILE__, __LINE__); + + ret = test_gcase_small(comm, iosysid, get_gcase_test_fname("small", iotype, rearr), iotype); + Util::check_spio_err(ret, "Testing G case failed", __FILE__, __LINE__); + + /* FIXME: Uncomment later, and optionally enable it based on user input */ + //ret = test_gcase_large(comm, iosysid, get_gcase_test_fname("large", iotype, rearr), iotype); + //Util::check_spio_err(ret, "Testing G case failed", __FILE__, __LINE__); + + PIOc_finalize(iosysid); + }); + + return PIO_NOERR; +} + diff --git a/examples/cxx/e3sm_i.cpp b/examples/cxx/e3sm_i.cpp new file mode 100644 index 0000000000..b5f4b5a317 --- /dev/null +++ b/examples/cxx/e3sm_i.cpp @@ -0,0 +1,160 @@ +#include "e3sm_fgi_utils.hpp" +#include "e3sm_fgi_data.hpp" + +/* Get unique output file name */ +static inline std::string get_icase_test_fname(const std::string &case_type, int iotype, int rearr) +{ + const std::string FNAME_PREFIX = "spio_e3sm_fgi_i"; + const std::string FNAME_SUFFIX = ".nc"; + const std::string SEP = "_"; + + return FNAME_PREFIX + + SEP + Util::GVars::iotype2str(iotype) + + SEP + Util::GVars::rearr2str(rearr) + + SEP + case_type + + FNAME_SUFFIX; +} + +/* Test pseudo E3SM I case */ +static int test_icase(MPI_Comm comm, int iosysid, const std::string &fname, int iotype, + int nlon, int nlat, int nframes) +{ + int comm_rank = 0, comm_sz = 0; + int ret = PIO_NOERR; + + + ret = MPI_Comm_rank(comm, &comm_rank); assert(ret == MPI_SUCCESS); + ret = MPI_Comm_size(comm, &comm_sz); assert(ret == MPI_SUCCESS); + + const bool is_last_proc = (comm_rank == comm_sz - 1) ? true : false; + int nlon_per_proc = std::max(nlon / comm_sz, 1); + int nlat_per_proc = nlat; + + int idx = 0, ilat = 0, ilon = 0; + std::function range_zero_to_inf = [idx] (void ) mutable { return static_cast(idx++); }; + + /* FIXME : Is this the I/O decomp used by the land model ?*/ + /* Simple 2D decomp, NLON s are divided evenly among all procs */ + std::function decomp_lon_2d_gen = + [comm_rank, is_last_proc, nlat, nlat_per_proc, nlon, nlon_per_proc, ilat, ilon](void ) mutable { + + const int ILON_START = comm_rank * nlon_per_proc; + + PIO_Offset val = ilat * nlon + ILON_START + ilon++ + 1; + + if(!is_last_proc){ + if(ilon >= nlon_per_proc) { ilon = 0; ilat++; } + } + else{ + if(ilon >= (nlon - (comm_rank * nlon_per_proc))) { ilon = 0; ilat++; } + } + + return val; + }; + + std::function decomp_lon_2d_val_gen = + [decomp_lon_2d_gen](void ){ + return static_cast(decomp_lon_2d_gen()); + }; + + /* Note: Last process gets the remaining NCOLs */ + int lsz = (!is_last_proc) ? (nlat_per_proc * nlon_per_proc) : + (nlat_per_proc * (nlon - comm_rank * nlon_per_proc)); + E3SM_FGI::SPIO_decomp decomp_lon_2d("decomp_lon_2d_lat_lon", iosysid, PIO_FLOAT, + std::vector({nlat, nlon}), lsz, + decomp_lon_2d_gen); + std::shared_ptr pdecomp_lon_2d = + std::make_shared(std::move(decomp_lon_2d)); + + E3SM_FGI::SPIO_file fh(iosysid, fname, iotype, false); + + /* Define vars/atts/dims */ + ret = fh.def({ + /* Global Attributes */ + std::make_shared("title", "ELM History file information"), + std::make_shared("source", "E3SM Land Model"), + std::make_shared("ltype_vegetated_or_bare_soil", 1), + std::make_shared("ltype_crop", 1), + /* Dimensions */ + std::make_shared("time", PIO_UNLIMITED), + std::make_shared("lat", nlat), + std::make_shared("lon", nlon), + /* Variables */ + std::make_shared >("time", std::vector({"time"}), std::vector({{"long_name", "time"}, {"units", "days since 0001-01-01 00:00:00"}, {"calendar", "no leap"}, {"bounds", "time_bounds"}}), range_zero_to_inf), + std::make_shared >("lat", std::vector({"lat"}), std::vector({{"long_name", "coordinate latitude"}, {"units", "degrees_north"}, {"_FillValue", (float ) -1.0}, {"missing_value", (float ) -1.0}}), range_zero_to_inf), + std::make_shared >("lon", std::vector({"lon"}), std::vector({{"long_name", "coordinate longitude"}, {"units", "degrees_east"}, {"_FillValue", (float ) -1.0}, {"missing_value", (float ) -1.0}}), range_zero_to_inf), + std::make_shared >("var_lat_lon", std::vector({"time", "lat", "lon"}), std::vector({}), pdecomp_lon_2d, nframes, decomp_lon_2d_val_gen) + }); + if(ret != PIO_NOERR){ + Util::GVars::logger->log(Util::Logging::LogLevel::ERROR, "Defining file objects failed"); + return ret; + } + + /* Write vars/atts */ + ret = fh.put(); + if(ret != PIO_NOERR){ + Util::GVars::logger->log(Util::Logging::LogLevel::ERROR, "Putting file objects failed"); + return ret; + } + + return ret; +} + +/* Large pseudo E3SM I case */ +static int test_icase_large(MPI_Comm comm, int iosysid, const std::string &fname, int iotype) +{ + const int NLON = 144; + const int NLAT = 96; + const int NFRAMES = 1; + + Util::GVars::logger->log(Util::Logging::LogLevel::STATUS, "Testing large I Case\n"); + return test_icase(comm, iosysid, fname, iotype, NLON, NLAT, NFRAMES); +} + +/* Small pseudo E3SM I case */ +static int test_icase_small(MPI_Comm comm, int iosysid, const std::string &fname, int iotype) +{ + const int NLON = 16; + const int NLAT = 8; + const int NFRAMES = 1; + + Util::GVars::logger->log(Util::Logging::LogLevel::STATUS, "Testing small I Case\n"); + return test_icase(comm, iosysid, fname, iotype, NLON, NLAT, NFRAMES); +} + +int E3SM_FGI::test_e3sm_icase(MPI_Comm comm, const std::vector &iotypes, + const std::vector &rearrs, int nioprocs) +{ + int comm_rank, comm_sz, ret; + ret = MPI_Comm_rank(comm, &comm_rank); assert(ret == MPI_SUCCESS); + ret = MPI_Comm_size(comm, &comm_sz); assert(ret == MPI_SUCCESS); + + assert(nioprocs <= comm_sz); + + int io_stride = comm_sz / nioprocs; + + /* Test I case for each combination of I/O types and I/O rearrangers */ + Util::zip_for_each(iotypes.cbegin(), iotypes.cend(), rearrs.cbegin(), rearrs.cend(), + [comm, nioprocs, io_stride](int iotype, int rearr){ + const int IOSYS_START_PROC = 0; + const int SPIO_IOSYSID_INVALID = -2; + int ret = PIO_NOERR; + + Util::GVars::logger->log(Util::Logging::LogLevel::STATUS, "Testing I Case, iotype = " + Util::GVars::iotype2str(iotype) + ", rearr = " + Util::GVars::rearr2str(rearr) + "\n"); + int iosysid = SPIO_IOSYSID_INVALID; + ret = PIOc_Init_Intracomm(comm, nioprocs, io_stride, IOSYS_START_PROC, + rearr, &iosysid); + Util::check_spio_err(ret, "PIOc_Init_Intracomm failed", __FILE__, __LINE__); + + ret = test_icase_small(comm, iosysid, get_icase_test_fname("small", iotype, rearr), iotype); + Util::check_spio_err(ret, "Testing I case (small) failed", __FILE__, __LINE__); + + ret = test_icase_large(comm, iosysid, get_icase_test_fname("large", iotype, rearr), iotype); + Util::check_spio_err(ret, "Testing I case (large) failed", __FILE__, __LINE__); + + PIOc_finalize(iosysid); + }); + + return PIO_NOERR; +} + diff --git a/src/clib/pio_config.h.in b/src/clib/pio_config.h.in index dc378d3c0d..efc781f13e 100644 --- a/src/clib/pio_config.h.in +++ b/src/clib/pio_config.h.in @@ -55,6 +55,9 @@ /** Extra bytes reserved in the header when creating NetCDF files. */ #define PIO_RESERVED_FILE_HEADER_SIZE @PIO_RESERVED_FILE_HEADER_SIZE@ +/** Set chunk size (in bytes) for HDF5/PnetCDF chunked variables. */ +#define PIO_CHUNK_SIZE @PIO_CHUNK_SIZE@ + /** Maximum number of I/O decompositions registered with ADIOS type */ #define PIO_MAX_ADIOS_DECOMPS @PIO_MAX_ADIOS_DECOMPS@ diff --git a/src/clib/pioc_support.cpp b/src/clib/pioc_support.cpp index d351a88769..f0cdd3e40d 100644 --- a/src/clib/pioc_support.cpp +++ b/src/clib/pioc_support.cpp @@ -17,6 +17,10 @@ #include #include #include +#include +#include +#include +#include #include "spio_io_summary.h" #include "spio_file_mvcache.h" #include "spio_hash.h" @@ -1164,6 +1168,10 @@ void piodie(const char *fname, int line, const char *fmt, ...) fprintf(stderr, " (%s: %d)\n", (fname) ? fname : "_", line); va_end(argp); +#ifdef _HDF5 + H5Eprint2(H5E_DEFAULT, stderr); +#endif + print_trace(stderr); #ifdef MPI_SERIAL abort(); @@ -1448,6 +1456,9 @@ int pio_err(iosystem_desc_t *ios, file_desc_t *file, if (print_err_msg) { fprintf(stderr, "PIO: ERROR: %s. %s (error num=%d), (%s:%d)\n", uerr_msg, err_msg, err_num, (fname) ? fname : "\0", line); +#ifdef _HDF5 + H5Eprint2(H5E_DEFAULT, stderr); +#endif fflush(stderr); } } @@ -6855,223 +6866,226 @@ static hid_t spio_create_hdf5_dataset_pid(iosystem_desc_t *ios, file_desc_t *fil #endif /* ifdef _SPIO_HDF5_USE_COMPRESSION */ + /* By default HDF5 does not track the order of creation of the attributes. So the attributes + * appear based on the alphanumeric order, of the attribute name, in the file. However, + * H5DSattach_scale calls (even with MPI_Barrier) might fail or hang if attribute creation + * order is tracked or indexed. Before we have a better workaround, temporarily disable + * tracking and indexing of attribute creation order. + */ +#if 0 + if(H5Pset_attr_creation_order(dcpl_id, H5P_CRT_ORDER_TRACKED | H5P_CRT_ORDER_INDEXED) < 0){ + return pio_err(ios, file, PIO_EHDF5ERR, __FILE__, __LINE__, + "Defining variable (%s, varid = %d) in file (%s, ncid=%d) using HDF5 iotype failed. " + "The low level (HDF5) I/O library call failed to set tracking and indexing of attribute creation order", + name, varid, pio_get_fname_from_file(file), file->pio_ncid); + } +#endif + return dpid; } -int spio_hdf5_def_var(iosystem_desc_t *ios, file_desc_t *file, const char *name, - nc_type xtype, int ndims, const int *dimidsp, int varid) +/* Get default chunk size (no of elems) - across each dimension - for variable data. The + * max chunk size (across all dimensions) is specified via PIO_CHUNK_SIZE (in bytes) + */ +static inline std::vector spio_get_dim_chunk_sz(const std::vector &dim_sz, nc_type xtype) { - hid_t h5_xtype; - hid_t sid; - hid_t dcpl_id = H5I_INVALID_HID; - hsize_t dims[H5S_MAX_RANK], mdims[H5S_MAX_RANK]; - int i; - - assert(ios && file && name && ndims >= 0 && varid >= 0); - assert((file->iotype == PIO_IOTYPE_HDF5) || (file->iotype == PIO_IOTYPE_HDF5C)); - assert(ios->ioproc); + std::size_t ndims = dim_sz.size(); + std::vector dim_chunk_sz = dim_sz; - for (i = 0; i < ndims; i++) - dims[i] = mdims[i] = file->hdf5_dims[dimidsp[i]].len; + /* Unlimited dimensions have a chunk size of 1 */ + std::transform(dim_chunk_sz.begin(), dim_chunk_sz.end(), dim_chunk_sz.begin(), + [](hsize_t i) { return (i != H5S_UNLIMITED) ? i : 1; }); - /* Create HDF5 dataset (and optionally add filters as needed) */ - dcpl_id = spio_create_hdf5_dataset_pid(ios, file, name, ndims, xtype); - if (dcpl_id == H5I_INVALID_HID) - { - return pio_err(ios, file, PIO_EHDF5ERR, __FILE__, __LINE__, - "Defining variable (%s, varid = %d) in file (%s, ncid=%d) using HDF5 iotype failed. " - "The low level (HDF5) I/O library call failed to create a new dataset creation property list", - name, varid, pio_get_fname_from_file(file), file->pio_ncid); - } + /* No chunking for scalars and 1D vars */ + if(ndims <= 1){ + return dim_chunk_sz; + } - /* H5DSattach_scale calls (even with MPI_Barrier) might fail or hang if attribute creation - * order is tracked or indexed. Before we have a better workaround, temporarily disable - * tracking and indexing of attribute creation order. */ -#if 0 - if (H5Pset_attr_creation_order(dcpl_id, H5P_CRT_ORDER_TRACKED | H5P_CRT_ORDER_INDEXED) < 0) - { - return pio_err(ios, file, PIO_EHDF5ERR, __FILE__, __LINE__, - "Defining variable (%s, varid = %d) in file (%s, ncid=%d) using HDF5 iotype failed. " - "The low level (HDF5) I/O library call failed to set tracking and indexing of attribute creation order", - name, varid, pio_get_fname_from_file(file), file->pio_ncid); - } -#endif + /* Number of elements corresponding to PIO_CHUNK_SIZE */ + double chunk_nelems = static_cast(PIO_CHUNK_SIZE)/static_cast(spio_get_nc_type_size(xtype)); + /* Assuming that elements are evenly distributed across all non-unlimited dimensions, + * Total (across all dimensions) chunked elements = (d * d * ...(n -1) times), where d is the size of each + * dimension + */ + hsize_t chunk_per_dim_nelems = static_cast(pow(chunk_nelems, 1.0/(ndims - 1) )); - if (xtype == NC_CHAR) - { - /* String type */ - h5_xtype = H5Tcopy(H5T_C_S1); - if (h5_xtype == H5I_INVALID_HID) - { - return pio_err(ios, file, PIO_EHDF5ERR, __FILE__, __LINE__, - "Defining variable (%s, varid = %d) in file (%s, ncid=%d) using HDF5 iotype failed. " - "The low level (HDF5) I/O library call failed to make a copy of the predefined string datatype in C", - name, varid, pio_get_fname_from_file(file), file->pio_ncid); - } + for(std::size_t i = 0; i < ndims; i++){ + /* Chunk size across UNLIMITED dimension is 1 */ + dim_chunk_sz[i] = (dim_sz[i] != H5S_UNLIMITED) ? (std::min(chunk_per_dim_nelems, dim_sz[i])) : 1; + } - if (H5Tset_strpad(h5_xtype, H5T_STR_NULLTERM) < 0) - { - return pio_err(ios, file, PIO_EHDF5ERR, __FILE__, __LINE__, - "Defining variable (%s, varid = %d) in file (%s, ncid=%d) using HDF5 iotype failed. " - "The low level (HDF5) I/O library call failed to define the type of padding (NULL-terminated) used for a derived C-style string datatype", - name, varid, pio_get_fname_from_file(file), file->pio_ncid); - } + return dim_chunk_sz; +} - if (H5Tset_cset(h5_xtype, H5T_CSET_ASCII) < 0) - { - return pio_err(ios, file, PIO_EHDF5ERR, __FILE__, __LINE__, - "Defining variable (%s, varid = %d) in file (%s, ncid=%d) using HDF5 iotype failed. " - "The low level (HDF5) I/O library call failed to set the character set (US ASCII) to be used in a derived C-style string datatype", - name, varid, pio_get_fname_from_file(file), file->pio_ncid); - } +/* Create an HDF5 string type - ASCII + null terminated */ +static inline hid_t spio_create_hdf5_str_type(void ) +{ + hid_t st = H5Tcopy(H5T_C_S1); + if(st != H5I_INVALID_HID){ + if(H5Tset_strpad(st, H5T_STR_NULLTERM) < 0){ + H5Tclose(st); + return H5I_INVALID_HID; } - else - { - h5_xtype = spio_nc_type_to_hdf5_type(xtype); - if (h5_xtype == H5I_INVALID_HID) - { - return pio_err(ios, file, PIO_EHDF5ERR, __FILE__, __LINE__, - "Defining variable (%s, varid = %d) in file (%s, ncid=%d) using HDF5 iotype failed. " - "Unsupported variable type (type=%x)", - name, varid, pio_get_fname_from_file(file), file->pio_ncid, xtype); - } + if(H5Tset_cset(st, H5T_CSET_ASCII) < 0){ + H5Tclose(st); + return H5I_INVALID_HID; } + } - file->hdf5_vars[varid].hdf5_type = h5_xtype; + return st; +} - if(ndims > 0){ - hsize_t cdim[H5S_MAX_RANK]; +/* Write a hidden coordinates attribute (_Netcdf4Coordinates), which lists the dimids of the variable. */ +static inline int spio_add_nc_hidden_coord(iosystem_desc_t *ios, file_desc_t *file, int varid, + int ndims, const int *dimidsp) +{ + assert(ios && file && (varid >= 0) && (ndims >= 0)); - for(i = 0; i < ndims; i++){ - cdim[i] = mdims[i]; - } + /* No coordinate atttribute for scalars */ + if(ndims == 0) return PIO_NOERR; - if(dims[0] == PIO_UNLIMITED){ - /* Chunk size along rec dim is always 1 */ - cdim[0] = 1; - } + assert(dimidsp); - if(H5Pset_chunk(dcpl_id, ndims, cdim) < 0){ - return pio_err(ios, file, PIO_EHDF5ERR, __FILE__, __LINE__, - "Defining variable (%s, varid = %d) in file (%s, ncid=%d) using HDF5 iotype failed. " - "The low level (HDF5) I/O library call failed to set the size of the chunks used to store a chunked layout dataset", - name, varid, pio_get_fname_from_file(file), file->pio_ncid); - } - } + /* Writing "_Netcdf4Coordinates" hidden attribute. This attribute stores the dimension ids of the + * variable dimensions in an integer array. + * This attribute is required for NetCDF to read HDF5 output + */ + const char* attr_name = "_Netcdf4Coordinates"; + + /* Sanity check : Ensure that "_Netcdf4Coordinates" attribute does not exist */ + htri_t attr_exists = H5Aexists(file->hdf5_vars[varid].hdf5_dataset_id, attr_name); + assert(attr_exists == 0); + + /* Create dataspace for the attribute i.e., space for integer array to store the dimension ids */ + std::array coords_len = {static_cast(ndims)}; + hid_t coords_space_id = H5Screate_simple(1, coords_len.data(), NULL); + if(coords_space_id == H5I_INVALID_HID){ + return pio_err(ios, file, PIO_EHDF5ERR, __FILE__, __LINE__, + "Defining variable (%s, varid = %d) in file (%s, ncid=%d) using HDF5 iotype failed. " + "The low level (HDF5) I/O library call failed to create a new simple dataspace", + pio_get_vname_from_file(file, varid), varid, pio_get_fname_from_file(file), file->pio_ncid); + } - if((ndims > 0) && (dims[0] == PIO_UNLIMITED)){ - mdims[0] = H5S_UNLIMITED; - } + /* Create the hidden attribute */ + hid_t coords_att_id = H5Acreate2(file->hdf5_vars[varid].hdf5_dataset_id, attr_name, + H5T_NATIVE_INT, coords_space_id, H5P_DEFAULT, H5P_DEFAULT); + if(coords_att_id == H5I_INVALID_HID){ + return pio_err(ios, file, PIO_EHDF5ERR, __FILE__, __LINE__, + "Defining variable (%s, varid = %d) in file (%s, ncid=%d) using HDF5 iotype failed. " + "The low level (HDF5) I/O library call failed to create a new attribute (%s) attached to the variable", + pio_get_vname_from_file(file, varid), varid, pio_get_fname_from_file(file), file->pio_ncid, attr_name); + } - sid = H5Screate_simple(ndims, dims, mdims); - if (sid == H5I_INVALID_HID) - { - return pio_err(ios, file, PIO_EHDF5ERR, __FILE__, __LINE__, - "Defining variable (%s, varid = %d) in file (%s, ncid=%d) using HDF5 iotype failed. " - "The low level (HDF5) I/O library call failed to create a new simple dataspace", - name, varid, pio_get_fname_from_file(file), file->pio_ncid); - } + /* Write the dimension ids */ + if(H5Awrite(coords_att_id, H5T_NATIVE_INT, dimidsp) < 0){ + return pio_err(ios, file, PIO_EHDF5ERR, __FILE__, __LINE__, + "Defining variable (%s, varid = %d) in file (%s, ncid=%d) using HDF5 iotype failed. " + "The low level (HDF5) I/O library call failed to write an attribute (%s) attached to the variable", + pio_get_vname_from_file(file, varid), varid, pio_get_fname_from_file(file), file->pio_ncid, attr_name); + } - const char* dataset_name = (file->hdf5_vars[varid].alt_name == NULL)? name : file->hdf5_vars[varid].alt_name; - file->hdf5_vars[varid].hdf5_dataset_id = H5Dcreate2(file->hdf5_file_id, dataset_name, h5_xtype, sid, H5P_DEFAULT, dcpl_id, H5P_DEFAULT); - if (file->hdf5_vars[varid].hdf5_dataset_id < 0) - { - return pio_err(ios, file, PIO_EHDF5ERR, __FILE__, __LINE__, - "Defining variable (%s, varid = %d) in file (%s, ncid=%d) using HDF5 iotype failed. " - "The low level (HDF5) I/O library call failed to create a new dataset (%s) for the variable", - name, varid, pio_get_fname_from_file(file), file->pio_ncid, dataset_name); - } + if(H5Aclose(coords_att_id) < 0){ + return pio_err(ios, file, PIO_EHDF5ERR, __FILE__, __LINE__, + "Defining variable (%s, varid = %d) in file (%s, ncid=%d) using HDF5 iotype failed. " + "The low level (HDF5) I/O library call failed to close an attribute (%s) attached to the variable", + pio_get_vname_from_file(file, varid), varid, pio_get_fname_from_file(file), file->pio_ncid, attr_name); + } - if (H5Sclose(sid) < 0) - { - return pio_err(ios, file, PIO_EHDF5ERR, __FILE__, __LINE__, - "Defining variable (%s, varid = %d) in file (%s, ncid=%d) using HDF5 iotype failed. " - "The low level (HDF5) I/O library call failed to release a simple dataspace", - name, varid, pio_get_fname_from_file(file), file->pio_ncid); - } + if(H5Sclose(coords_space_id) < 0){ + return pio_err(ios, file, PIO_EHDF5ERR, __FILE__, __LINE__, + "Defining variable (%s, varid = %d) in file (%s, ncid=%d) using HDF5 iotype failed. " + "The low level (HDF5) I/O library call failed to release a simple dataspace", + pio_get_vname_from_file(file, varid), varid, pio_get_fname_from_file(file), file->pio_ncid); + } - if (H5Pclose(dcpl_id) < 0) - { - return pio_err(ios, file, PIO_EHDF5ERR, __FILE__, __LINE__, - "Defining variable (%s, varid = %d) in file (%s, ncid=%d) using HDF5 iotype failed. " - "The low level (HDF5) I/O library call failed to close a dataset creation property list", - name, varid, pio_get_fname_from_file(file), file->pio_ncid); + return PIO_NOERR; +} + +int spio_hdf5_def_var(iosystem_desc_t *ios, file_desc_t *file, const char *name, + nc_type xtype, int ndims, const int *dimidsp, int varid) +{ + assert(ios && file && name && ndims >= 0 && varid >= 0); + assert((ndims == 0) || dimidsp); + assert((file->iotype == PIO_IOTYPE_HDF5) || (file->iotype == PIO_IOTYPE_HDF5C)); + assert(ios->ioproc); + + /* Cache the dim sizes for HDF5 calls */ + std::vector dim_sz(ndims), max_dim_sz(ndims); + for(int i = 0; i < ndims; i++){ + if(file->hdf5_dims[dimidsp[i]].len != PIO_UNLIMITED){ + dim_sz[i] = max_dim_sz[i] = file->hdf5_dims[dimidsp[i]].len; + } + else{ + dim_sz[i] = 1; + max_dim_sz[i] = H5S_UNLIMITED; } + } - /* Write a hidden coordinates attribute (_Netcdf4Coordinates), which lists the dimids of this variable. */ - if (ndims > 0) - { - hsize_t coords_len[1]; - hid_t coords_space_id, coords_att_id; - htri_t attr_exists; + /* Create HDF5 dataset (and optionally add filters as needed) */ + hid_t dcpl_id = spio_create_hdf5_dataset_pid(ios, file, name, ndims, xtype); + if(dcpl_id == H5I_INVALID_HID){ + return pio_err(ios, file, PIO_EHDF5ERR, __FILE__, __LINE__, + "Defining variable (%s, varid = %d) in file (%s, ncid=%d) using HDF5 iotype failed. " + "The low level (HDF5) I/O library call failed to create a new dataset creation property list", + name, varid, pio_get_fname_from_file(file), file->pio_ncid); + } - coords_len[0] = ndims; - coords_space_id = H5Screate_simple(1, coords_len, coords_len); - if (coords_space_id == H5I_INVALID_HID) - { - return pio_err(ios, file, PIO_EHDF5ERR, __FILE__, __LINE__, - "Defining variable (%s, varid = %d) in file (%s, ncid=%d) using HDF5 iotype failed. " - "The low level (HDF5) I/O library call failed to create a new simple dataspace", - name, varid, pio_get_fname_from_file(file), file->pio_ncid); - } + file->hdf5_vars[varid].hdf5_type = (xtype == NC_CHAR) ? spio_create_hdf5_str_type() : spio_nc_type_to_hdf5_type(xtype); + assert(file->hdf5_vars[varid].hdf5_type != H5I_INVALID_HID); - /* Writing _Netcdf4Coordinates attribute */ - const char* attr_name = "_Netcdf4Coordinates"; + /* Set default chunk size for variable data */ + if(ndims > 0){ + if(H5Pset_chunk(dcpl_id, ndims, spio_get_dim_chunk_sz(max_dim_sz, xtype).data()) < 0){ + return pio_err(ios, file, PIO_EHDF5ERR, __FILE__, __LINE__, + "Defining variable (%s, varid = %d) in file (%s, ncid=%d) using HDF5 iotype failed. " + "The low level (HDF5) I/O library call failed to set the size of the chunks used to store a chunked layout dataset", + name, varid, pio_get_fname_from_file(file), file->pio_ncid); + } + } - /* H5Aexists() returns zero (false), a positive (true) or a negative (failure) value */ - attr_exists = H5Aexists(file->hdf5_vars[varid].hdf5_dataset_id, attr_name); - if (attr_exists > 0) - { - assert(0); - } - else if (attr_exists == 0) - { - coords_att_id = H5Acreate2(file->hdf5_vars[varid].hdf5_dataset_id, attr_name, - H5T_NATIVE_INT, coords_space_id, H5P_DEFAULT, H5P_DEFAULT); - if (coords_att_id == H5I_INVALID_HID) - { - return pio_err(ios, file, PIO_EHDF5ERR, __FILE__, __LINE__, - "Defining variable (%s, varid = %d) in file (%s, ncid=%d) using HDF5 iotype failed. " - "The low level (HDF5) I/O library call failed to create a new attribute (%s) attached to the variable", - name, varid, pio_get_fname_from_file(file), file->pio_ncid, attr_name); - } - } - else - { - /* Error determining whether an attribute with a given name exists on an object */ - return pio_err(ios, file, PIO_EHDF5ERR, __FILE__, __LINE__, - "Defining variable (%s, varid = %d) in file (%s, ncid=%d) using HDF5 iotype failed. " - "The low level (HDF5) I/O library call failed to determine whether an attribute (%s) exists on the variable", - name, varid, pio_get_fname_from_file(file), file->pio_ncid, attr_name); - } + /* Create a simple dataspace to define the global variable dimensions */ + hid_t sid = H5Screate_simple(ndims, dim_sz.data(), max_dim_sz.data()); + if(sid == H5I_INVALID_HID){ + return pio_err(ios, file, PIO_EHDF5ERR, __FILE__, __LINE__, + "Defining variable (%s, varid = %d) in file (%s, ncid=%d) using HDF5 iotype failed. " + "The low level (HDF5) I/O library call failed to create a new simple dataspace", + name, varid, pio_get_fname_from_file(file), file->pio_ncid); + } - assert(dimidsp); - if (H5Awrite(coords_att_id, H5T_NATIVE_INT, dimidsp) < 0) - { - return pio_err(ios, file, PIO_EHDF5ERR, __FILE__, __LINE__, - "Defining variable (%s, varid = %d) in file (%s, ncid=%d) using HDF5 iotype failed. " - "The low level (HDF5) I/O library call failed to write an attribute (%s) attached to the variable", - name, varid, pio_get_fname_from_file(file), file->pio_ncid, attr_name); - } + /* Define the variable */ + const char* dataset_name = (file->hdf5_vars[varid].alt_name == NULL)? name : file->hdf5_vars[varid].alt_name; + file->hdf5_vars[varid].hdf5_dataset_id = H5Dcreate2(file->hdf5_file_id, dataset_name, file->hdf5_vars[varid].hdf5_type, + sid, H5P_DEFAULT, dcpl_id, H5P_DEFAULT); + if(file->hdf5_vars[varid].hdf5_dataset_id < 0){ + return pio_err(ios, file, PIO_EHDF5ERR, __FILE__, __LINE__, + "Defining variable (%s, varid = %d) in file (%s, ncid=%d) using HDF5 iotype failed. " + "The low level (HDF5) I/O library call failed to create a new dataset (%s) for the variable", + name, varid, pio_get_fname_from_file(file), file->pio_ncid, dataset_name); + } - if (H5Sclose(coords_space_id) < 0) - { - return pio_err(ios, file, PIO_EHDF5ERR, __FILE__, __LINE__, - "Defining variable (%s, varid = %d) in file (%s, ncid=%d) using HDF5 iotype failed. " - "The low level (HDF5) I/O library call failed to release a simple dataspace", - name, varid, pio_get_fname_from_file(file), file->pio_ncid); - } + if(H5Sclose(sid) < 0){ + return pio_err(ios, file, PIO_EHDF5ERR, __FILE__, __LINE__, + "Defining variable (%s, varid = %d) in file (%s, ncid=%d) using HDF5 iotype failed. " + "The low level (HDF5) I/O library call failed to release a simple dataspace", + name, varid, pio_get_fname_from_file(file), file->pio_ncid); + } - if (H5Aclose(coords_att_id) < 0) - { - return pio_err(ios, file, PIO_EHDF5ERR, __FILE__, __LINE__, - "Defining variable (%s, varid = %d) in file (%s, ncid=%d) using HDF5 iotype failed. " - "The low level (HDF5) I/O library call failed to close an attribute (%s) attached to the variable", - name, varid, pio_get_fname_from_file(file), file->pio_ncid, attr_name); - } - } + if(H5Pclose(dcpl_id) < 0){ + return pio_err(ios, file, PIO_EHDF5ERR, __FILE__, __LINE__, + "Defining variable (%s, varid = %d) in file (%s, ncid=%d) using HDF5 iotype failed. " + "The low level (HDF5) I/O library call failed to close a dataset creation property list", + name, varid, pio_get_fname_from_file(file), file->pio_ncid); + } - return PIO_NOERR; + /* Add a hidden attribute, "_Netcdf4Coordinates", to store var dimension ids so that NetCDF can read the var */ + if(spio_add_nc_hidden_coord(ios, file, varid, ndims, dimidsp) != PIO_NOERR){ + return pio_err(ios, file, PIO_EHDF5ERR, __FILE__, __LINE__, + "Defining variable (%s, varid = %d) in file (%s, ncid=%d) using HDF5 iotype failed. " + "Adding NetCDF hidden coordinate attribute failed", + pio_get_vname_from_file(file, varid), varid, pio_get_fname_from_file(file), file->pio_ncid); + } + + return PIO_NOERR; } int spio_hdf5_enddef(iosystem_desc_t *ios, file_desc_t *file) diff --git a/tests/general/CMakeLists.txt b/tests/general/CMakeLists.txt index aa3abb7f9c..fbaf4eb310 100644 --- a/tests/general/CMakeLists.txt +++ b/tests/general/CMakeLists.txt @@ -90,6 +90,7 @@ if (CMAKE_Fortran_COMPILER_ID STREQUAL "NAG") endif () if (PIO_TEST_CLOSE_OPEN_FOR_SYNC) + message(STATUS "Turning on Close+Open for syncing file output in tests") add_definitions(-DPIO_TEST_CLOSE_OPEN_FOR_SYNC) endif() diff --git a/util/argparser.cxx b/util/argparser.cxx index b9b7ef5cf3..06a947dcc9 100644 --- a/util/argparser.cxx +++ b/util/argparser.cxx @@ -65,9 +65,8 @@ void ArgParser::parse(int argc, char *argv[]) /* The "--help" option is provided by default */ if ((argvi == "--help") || (argvi == "-h")){ print_usage(std::cout); - return; } - else if (std::regex_search(argvi, noval_match, noval_opt_rgx) && + if (std::regex_search(argvi, noval_match, noval_opt_rgx) && (noval_match.size() == 2)){ /* No value arguments like "--verbose" */ if (opts_map_.count(noval_match.str(1)) != 1){