Merge pull request #647 from E3SM-Project/jayeshkrishna/add_rm_after_conv

Adding a new command line argument in the ADIOS to
NetCDF conversion tool to specify whether (and which)
to delete input files (ADIOS BP) after converting
to NetCDF

Also including a simple README to document usage of
the conversion tool

Author: jayeshkrishna

tools/adios2pio-nm/CMakeLists.txt

@@ -89,7 +89,7 @@ endif ()
 #  DEFINE THE TARGET LIBRARY
 #==============================================================================
 SET(SRC ${SCORPIO_SOURCE_DIR}/tools/util/argparser.cxx adios2pio-nm.cxx)
-add_library(adios2pio-nm-lib adios2pio-nm-lib.cxx)
+add_library(adios2pio-nm-lib ${SCORPIO_SOURCE_DIR}/tools/util/spio_misc_tool_utils.cxx adios2pio-nm-lib.cxx)
 include_directories(
   "${PROJECT_SOURCE_DIR}"   # to find foo/foo.h
   "${PROJECT_BINARY_DIR}")  # to find foo/config.h

tools/adios2pio-nm/README_conversion_tool.txt

@@ -0,0 +1,54 @@
+This file contains some basic information on how to use the ADIOS to NetCDF conversion tool for converting SCORPIO output files in the ADIOS BP file format to NetCDF. The conversion tool reads the ADIOS BP files in parallel (using SCORPIO and ADIOS) and converts the file using SCORPIO and PnetCDF or NetCDF libraries.
+
+Running the application
+------------------------
+If SCORPIO is enabled with support for ADIOS (see the configure log of SCORPIO to see if ADIOS support is enabled), using PIO_IOTYPE_ADIOS (for uncompressed output) or PIO_IOTYPE_ADIOSC (for compressed output) will result in all SCORPIO output being written in the ADIOS BP file format. The output files in the ADIOS BP file format can be converted to NetCDF using the conversion tool in a separate batch job.
+
+The conversion tool is an MPI program and can be run with different number (i.e., fewer) of MPI processes compared to the application that generated the output.
+
+
+---------------------------------------------------------------------------------------
+EXAMPLES:
+---------------------------------------------------------------------------------------
+
+# Program options
+
+jayesh@compute-386-03:~/scorpio_build2$ ./tools/adios2pio-nm/adios2pio-nm.exe
+Usage : ./tools/adios2pio-nm/adios2pio-nm.exe --[OPTIONAL ARG1 NAME]=[OPTIONAL ARG1 VALUE] --[OPTIONAL ARG2 NAME]=[OPTIONAL ARG2 VALUE] ... 
+Optional Arguments :
+--bp-file:	data produced by SCORPIO with ADIOS format
+--idir:	Directory containing data output from SCORPIO (in ADIOS format)
+--nc-file:	output file name after conversion
+--pio-format:	output SCORPIO I/O type. Supported parameters: "pnetcdf",  "netcdf",  "netcdf4c",  "netcdf4p", "nczarr"
+--rearr:	SCORPIO rearranger. Supported parameters: "subset", "box", "any". Default "any".
+--rm-bp-file-after-conv:	remove the ADIOS BP file after conversion
+--verbose:	Turn on verbose info messages
+
+
+# Delete all ADIOS BP files after conversion
+jayesh@compute-386-03:~/scorpio_build2$ ./tools/adios2pio-nm/adios2pio-nm.exe --bp-file=/home/jayesh/scorpio_build2/tests/general/test_pio_file_simple_tests.testfile.bp --rm-bp-file-after-conv
+Converting BP file : "/home/jayesh/scorpio_build2/tests/general/test_pio_file_simple_tests.testfile.bp"
+Deleting BP file : /home/jayesh/scorpio_build2/tests/general/test_pio_file_simple_tests.testfile.bp
+
+# Delete all ADIOS BP files with name "(.*)(test_pio_file_simple)(.*)" after conversion
+jayesh@compute-386-03:~/scorpio_build2$ ./tools/adios2pio-nm/adios2pio-nm.exe --bp-file=/home/jayesh/scorpio_build2/tests/general/test_pio_file_simple_tests.testfile.bp --rm-bp-file-after-conv="(.*)(test_pio_file_simple)(.*)"
+Converting BP file : "/home/jayesh/scorpio_build2/tests/general/test_pio_file_simple_tests.testfile.bp"
+Deleting BP file : /home/jayesh/scorpio_build2/tests/general/test_pio_file_simple_tests.testfile.bp
+
+jayesh@compute-386-03:~/scorpio_build2$ ./tools/adios2pio-nm/adios2pio-nm.exe --bp-file=/home/jayesh/scorpio_build2/tests/general/test_pio_file_simple_tests.testfile.bp --rm-bp-file-after-conv="(.*)(DONT_DELETE_test_pio_file_simple)(.*)"
+Converting BP file : "/home/jayesh/scorpio_build2/tests/general/test_pio_file_simple_tests.testfile.bp"
+
+# Convert ADIOS BP output files in directory, /scratch/jayesh/e3sm/scratch/test_F2010_ne4_oQU240/run, and only delete the EAM ADIOS BP files after conversion
+jayesh@compute-386-03:~/scorpio_build2$ mpiexec -n 8 ./tools/adios2pio-nm/adios2pio-nm.exe --idir=/scratch/jayesh/e3sm/scratch/test_F2010_ne4_oQU240/run --rm-bp-file-after-conv="(.*)(.eam.)(.*)"
+Converting BP file : "/scratch/jayesh/e3sm/scratch/test_F2010_ne4_oQU240/run/test_F2010_ne4_oQU240.eam.rs.0001-01-02-00000.nc.bp"
+Deleting BP file : /scratch/jayesh/e3sm/scratch/test_F2010_ne4_oQU240/run/test_F2010_ne4_oQU240.eam.rs.0001-01-02-00000.nc.bp
+Converting BP file : "/scratch/jayesh/e3sm/scratch/test_F2010_ne4_oQU240/run/test_F2010_ne4_oQU240.mosart.rh0.0001-01-02-00000.nc.bp"
+Converting BP file : "/scratch/jayesh/e3sm/scratch/test_F2010_ne4_oQU240/run/test_F2010_ne4_oQU240.elm.rh0.0001-01-02-00000.nc.bp"
+Converting BP file : "/scratch/jayesh/e3sm/scratch/test_F2010_ne4_oQU240/run/test_F2010_ne4_oQU240.cpl.r.0001-01-02-00000.nc.bp"
+Converting BP file : "/scratch/jayesh/e3sm/scratch/test_F2010_ne4_oQU240/run/test_F2010_ne4_oQU240.elm.r.0001-01-02-00000.nc.bp"
+Converting BP file : "/scratch/jayesh/e3sm/scratch/test_F2010_ne4_oQU240/run/test_F2010_ne4_oQU240.eam.h0.0001-01-01-00000.nc.bp"
+Deleting BP file : /scratch/jayesh/e3sm/scratch/test_F2010_ne4_oQU240/run/test_F2010_ne4_oQU240.eam.h0.0001-01-01-00000.nc.bp
+Converting BP file : "/scratch/jayesh/e3sm/scratch/test_F2010_ne4_oQU240/run/test_F2010_ne4_oQU240.eam.r.0001-01-02-00000.nc.bp"
+Deleting BP file : /scratch/jayesh/e3sm/scratch/test_F2010_ne4_oQU240/run/test_F2010_ne4_oQU240.eam.r.0001-01-02-00000.nc.bp
+Converting BP file : "/scratch/jayesh/e3sm/scratch/test_F2010_ne4_oQU240/run/test_F2010_ne4_oQU240.mosart.r.0001-01-02-00000.nc.bp"
+

tools/adios2pio-nm/adios2pio-nm-lib.cxx

@@ -19,6 +19,7 @@
 
 #include "adios2pio-nm-lib.h"
 #include "adios2pio-nm-lib-c.h"
+#include "spio_misc_tool_utils.h"
 
 using namespace std;
 
@@ -2305,12 +2306,14 @@ int ConvertBPFile(const string &infilepath, const string &outfilename,
     if(ierr != BP2PIO_NOERR)
     {
         fprintf(stderr, "ERROR: Checking BP file (%s) conversion status failed\n", infilepath.c_str());
+        GPTLstop("adios2pio:ConvertBPFile");
         return BP2PIO_ERROR;
     }
 
     if(conv_done)
     {
         /* Avoid converting BP files that have already been converted */
+        GPTLstop("adios2pio:ConvertBPFile");
         return BP2PIO_NOERR;
     }
 
@@ -2325,6 +2328,7 @@ int ConvertBPFile(const string &infilepath, const string &outfilename,
         ierr = OpenAdiosFile(adios, bpIO, bpReader, file0, err_msg);
         if (ierr != PIO_NOERR)
         {
+            GPTLstop("adios2pio:ConvertBPFile");
             fprintf(stderr, "ERROR: Cannot open file: %s\n", file0.c_str());
             fflush(stderr);
             return ierr;
@@ -2343,6 +2347,7 @@ int ConvertBPFile(const string &infilepath, const string &outfilename,
         else
         {
             fprintf(stderr, "ERROR: /__pio__/info/nproc is missing.\n");
+            GPTLstop("adios2pio:ConvertBPFile");
             return BP2PIO_ERROR;
         }
 
@@ -2396,12 +2401,14 @@ int ConvertBPFile(const string &infilepath, const string &outfilename,
             if (mpierr != MPI_SUCCESS)
             {
                 fprintf(stderr, "Bcasting /__pio__/info/block_nprocs failed, mpierr = %d\n", mpierr);
+                GPTLstop("adios2pio:ConvertBPFile");
                 return BP2PIO_ERROR;
             }
         }
         else
         {
             fprintf(stderr, "ERROR: /__pio__/info/block_nprocs is missing.\n");
+            GPTLstop("adios2pio:ConvertBPFile");
             return BP2PIO_ERROR;
         }
 
@@ -2443,6 +2450,7 @@ int ConvertBPFile(const string &infilepath, const string &outfilename,
             if (mpierr != MPI_SUCCESS)
             {
                 fprintf(stderr, "Bcasting /__pio__/info/block_list failed, mpierr = %d\n", mpierr);
+                GPTLstop("adios2pio:ConvertBPFile");
                 return BP2PIO_ERROR;
             }
 
@@ -2454,6 +2462,7 @@ int ConvertBPFile(const string &infilepath, const string &outfilename,
         else
         {
             fprintf(stderr, "ERROR: /__pio__/info/block_list is missing.\n");
+            GPTLstop("adios2pio:ConvertBPFile");
             return BP2PIO_ERROR;
         }
         bpReader[0].EndStep();
@@ -2464,7 +2473,10 @@ int ConvertBPFile(const string &infilepath, const string &outfilename,
         ierr = CreateIOProcessGroup(w_comm, w_nproc, w_mpirank, block_procs, &comm, &mpirank, &nproc, &io_proc);
         GPTLstop("adios2pio:ConvertBPFile:CreateIOProcessGroup");
         if (ierr != BP2PIO_NOERR)
+        {
+            GPTLstop("adios2pio:ConvertBPFile");
             return ierr;
+        }
 
         /* Close files. Create a new ADIOS object, because the MPI processes are clustered into two groups */
         bpReader[0].Close();
@@ -2479,6 +2491,7 @@ int ConvertBPFile(const string &infilepath, const string &outfilename,
             /* Not I/O process */
             MPI_Comm_free(&comm);
             MPI_Barrier(w_comm);
+            GPTLstop("adios2pio:ConvertBPFile");
             return BP2PIO_NOERR;
         }
 
@@ -2660,6 +2673,7 @@ int ConvertBPFile(const string &infilepath, const string &outfilename,
         {
             cerr << "rank " << mpirank << ":ERROR in PIOc_finalize(), code = " << ret
                  << " at " << __func__ << ":" << __LINE__ << endl;
+           GPTLstop("adios2pio:ConvertBPFile");
            throw std::runtime_error("PIOc_finalize error.");
         }
     }
@@ -2683,7 +2697,9 @@ int ConvertBPFile(const string &infilepath, const string &outfilename,
     GPTLstop("adios2pio:ConvertBPFile");
 
     if (ierr != BP2PIO_NOERR)
+    {
         return ierr;
+    }
 
     return BP2PIO_NOERR;
 }
@@ -2720,9 +2736,13 @@ enum PIO_IOTYPE GetIOType_nm(const string &t)
 }
 
 int ConvertBPToNC(const string &infilepath, const string &outfilename,
-                  const string &piotype, const string &rearr, MPI_Comm comm_in)
+                  const string &piotype, const string &rearr, MPI_Comm comm_in,
+                  const std::string &rm_ifname_rgx)
 {
     int ierr = BP2PIO_NOERR;
+    int rank = -1;
+
+    MPI_Comm_rank(comm_in, &rank);
 
     try
     {
@@ -2732,6 +2752,23 @@ int ConvertBPToNC(const string &infilepath, const string &outfilename,
         {
             throw std::runtime_error("ConvertBPFile error.");
         }
+
+        if ((rank == 0) && !rm_ifname_rgx.empty())
+        {
+            /* On root proc, delete the input BP file if its name matches the passed in regex */
+#ifdef SPIO_NO_CXX_REGEX
+            /* Only "*" is supported for no regex case, and we assume its "*" if not empty() */
+            spio_tool_utils::rmdir_f(infilepath);
+#else
+            std::smatch match;
+            std::regex rgx(rm_ifname_rgx);
+            if (std::regex_match(infilepath, match, rgx))
+            {
+                std::cout << "Deleting BP file : " << infilepath.c_str() << "\n";
+                spio_tool_utils::rmdir_f(infilepath);
+            }
+#endif
+        }
     }
     catch (const std::exception &e)
     {
@@ -2809,7 +2846,7 @@ static int FindBPDirs(const string &bppdir,
     while ((pde = readdir(pdir)) != NULL)
     {
         assert(pde);
-        string dname(pde->d_name);
+        string dname(bppdir + "/" + pde->d_name);
         /* Add dirs named "*.bp" to bpdirs */
         string dname_prefix;
         if ((pde->d_type == DT_DIR) &&
@@ -2838,7 +2875,7 @@ static int FindBPDirs(const string &bppdir,
  * and converts them, one at a time, to NetCDF files
  */
 int MConvertBPToNC(const string &bppdir, const string &piotype, const string &rearr,
-                    MPI_Comm comm)
+                    MPI_Comm comm, const std::string &rm_ifname_rgx)
 {
     int ierr = BP2PIO_NOERR;
     vector<string> bpdirs;
@@ -2858,7 +2895,7 @@ int MConvertBPToNC(const string &bppdir, const string &piotype, const string &re
         MPI_Barrier(comm);
         ierr = ConvertBPToNC(bpdirs[i],
                 conv_fname_prefixes[i] + CONV_FNAME_SUFFIX,
-                piotype, rearr, comm);
+                piotype, rearr, comm, rm_ifname_rgx);
         MPI_Barrier(comm);
         if (ierr != BP2PIO_NOERR)
         {
@@ -2884,8 +2921,10 @@ int C_API_ConvertBPToNC(const char *infilepath, const char *outfilename,
     else
         rearr = "subset";
 
+    /* FIXME: Currently we don't support regex deletes of BP input files */
+    std::string rm_ifname_rgx;
     return ConvertBPToNC(string(infilepath), string(outfilename),
-                         string(piotype), rearr, comm_in);
+                         string(piotype), rearr, comm_in, rm_ifname_rgx);
 }
 
 #ifdef __cplusplus

tools/adios2pio-nm/adios2pio-nm-lib.h

@@ -17,9 +17,10 @@ using namespace std;
 
 int ConvertBPToNC(const string &infilepath,
                   const string &outfilename,
-                  const string &piotype, const string &rearr, MPI_Comm comm_in);
+                  const string &piotype, const string &rearr, MPI_Comm comm_in,
+                  const std::string &rm_ifname_rgx);
 int MConvertBPToNC(const string &bppdir, const string &piotype, const string &rearr,
-                    MPI_Comm comm);
+                    MPI_Comm comm, const std::string &rm_ifname_rgx);
 void SetDebugOutput(int val);
 
 #endif /* #ifndef _ADIOS2PIO_NM_LIB_H_ */

tools/adios2pio-nm/adios2pio-nm.cxx

@@ -9,11 +9,12 @@
 
 static void init_user_options(spio_tool_utils::ArgParser &ap)
 {
-    ap.add_opt("bp-file", "data produced by PIO with ADIOS format")
-      .add_opt("idir", "Directory containing data output from PIO (in ADIOS format)")
+    ap.add_opt("bp-file", "data produced by SCORPIO with ADIOS format")
+      .add_opt("rm-bp-file-after-conv", "remove the ADIOS BP file after conversion")
+      .add_opt("idir", "Directory containing data output from SCORPIO (in ADIOS format)")
       .add_opt("nc-file", "output file name after conversion")
-      .add_opt("pio-format", "output PIO_IO_TYPE. Supported parameters: \"pnetcdf\",  \"netcdf\",  \"netcdf4c\",  \"netcdf4p\", \"nczarr\"")
-      .add_opt("rearr", "PIO rearranger. Supported parameters: \"subset\", \"box\", \"any\". Default \"any\".")
+      .add_opt("pio-format", "output SCORPIO I/O type. Supported parameters: \"pnetcdf\",  \"netcdf\",  \"netcdf4c\",  \"netcdf4p\", \"nczarr\"")
+      .add_opt("rearr", "SCORPIO rearranger. Supported parameters: \"subset\", \"box\", \"any\". Default \"any\".")
       .add_opt("verbose", "Turn on verbose info messages");
 }
 
@@ -55,9 +56,10 @@ static int get_user_options(
               int argc, char *argv[],
               std::string &idir,
               std::string &ifile, std::string &ofile,
+              std::string &rm_ifname_rgx,
               std::string &otype,
               std::string &rearr,
-              int &debug_lvl)
+              int &debug_lvl, int rank)
 {
     const std::string DEFAULT_PIO_FORMAT("pnetcdf");
     const std::string DEFAULT_REARRANGER("any");
@@ -96,6 +98,23 @@ static int get_user_options(
         idir = ap.get_arg<std::string>("idir");
     }
 
+    if (ap.has_arg("rm-bp-file-after-conv"))
+    {
+        try{
+          /* Remove all input files matching the provided regex */
+          rm_ifname_rgx = ap.get_arg<std::string>("rm-bp-file-after-conv");
+#ifdef SPIO_NO_CXX_REGEX
+          if(rank == 0){
+            std::cout << "WARNING: No C++ regex support. Ignoring specified regex for deleting files and deleting all input/BP files after conversion...\n";
+          }
+          rm_ifname_rgx = ".*";
+#endif
+        } catch(...){
+          /* Remove all input files */
+          rm_ifname_rgx = ".*";
+        }
+    }
+
     if (ap.has_arg("pio-format"))
     {
         otype = ap.get_arg<std::string>("pio-format");
@@ -143,10 +162,11 @@ int main(int argc, char *argv[])
 
     /* Parse the user options */
     string idir, infilepath, outfilename, piotype, rearr;
+    string rm_ifname_rgx;
     int debug_lvl = 0;
     ret = get_user_options(ap, argc, argv,
                             idir, infilepath, outfilename,
-                            piotype, rearr, debug_lvl);
+                            rm_ifname_rgx, piotype, rearr, debug_lvl, rank);
 
     if (ret != 0)
     {
@@ -165,11 +185,11 @@ int main(int argc, char *argv[])
     MPI_Barrier(comm_in);
     if (idir.size() == 0)
     {
-        ret = ConvertBPToNC(infilepath, outfilename, piotype, rearr, comm_in);
+        ret = ConvertBPToNC(infilepath, outfilename, piotype, rearr, comm_in, rm_ifname_rgx);
     }
     else
     {
-        ret = MConvertBPToNC(idir, piotype, rearr, comm_in);
+        ret = MConvertBPToNC(idir, piotype, rearr, comm_in, rm_ifname_rgx);
     }
     MPI_Barrier(comm_in);
 

tools/util/argparser.h

@@ -80,7 +80,11 @@ template<>
 inline std::string ArgParser::get_arg<std::string>(const std::string &opt) const
 {
   assert(arg_map_.count(opt) == 1);
-  return arg_map_.at(opt);
+  std::string opt_val = arg_map_.at(opt);
+  if(opt_val == NOVAL_OPT_STR){
+    throw std::runtime_error("Requested value for argument with no value");
+  }
+  return opt_val;
 }
 
 /* Get an int argument already parsed by the parser */