Support specifying files to convert via regex

Users can specify the basename of the files to be converted using
a regex via the "--bp-file" option. Only files with basenames that
match the regex are converted.

Author: jayeshkrishna

tools/adios2pio-nm/README_conversion_tool.txt

@@ -13,18 +13,17 @@ EXAMPLES:
 
 # Program options
 
-jayesh@compute-386-03:~/scorpio_build2$ ./tools/adios2pio-nm/adios2pio-nm.exe
+jayesh@compute-386-03:~/scorpio_build2$ ./tools/adios2pio-nm/adios2pio-nm.exe --help
 Usage : ./tools/adios2pio-nm/adios2pio-nm.exe --[OPTIONAL ARG1 NAME]=[OPTIONAL ARG1 VALUE] --[OPTIONAL ARG2 NAME]=[OPTIONAL ARG2 VALUE] ... 
 Optional Arguments :
---bp-file:	data produced by SCORPIO with ADIOS format
---idir:	Directory containing data output from SCORPIO (in ADIOS format)
---nc-file:	output file name after conversion
+--bp-file:	input file (or regex of basename for multiple files) in ADIOS BP format
+--idir:	Directory containing the input files (in ADIOS BP format)
+--nc-file:	output file name after conversion (will be used as prefix when converting multiple files)
 --pio-format:	output SCORPIO I/O type. Supported parameters: "pnetcdf",  "netcdf",  "netcdf4c",  "netcdf4p", "nczarr"
 --rearr:	SCORPIO rearranger. Supported parameters: "subset", "box", "any". Default "any".
---rm-bp-file-after-conv:	remove the ADIOS BP file after conversion
+--rm-bp-file-after-conv:	remove the ADIOS BP file after conversion (or regex to specify which files to delete after conversion)
 --verbose:	Turn on verbose info messages
 
-
 # Delete all ADIOS BP files after conversion
 jayesh@compute-386-03:~/scorpio_build2$ ./tools/adios2pio-nm/adios2pio-nm.exe --bp-file=/home/jayesh/scorpio_build2/tests/general/test_pio_file_simple_tests.testfile.bp --rm-bp-file-after-conv
 Converting BP file : "/home/jayesh/scorpio_build2/tests/general/test_pio_file_simple_tests.testfile.bp"
@@ -52,3 +51,16 @@ Converting BP file : "/scratch/jayesh/e3sm/scratch/test_F2010_ne4_oQU240/run/tes
 Deleting BP file : /scratch/jayesh/e3sm/scratch/test_F2010_ne4_oQU240/run/test_F2010_ne4_oQU240.eam.r.0001-01-02-00000.nc.bp
 Converting BP file : "/scratch/jayesh/e3sm/scratch/test_F2010_ne4_oQU240/run/test_F2010_ne4_oQU240.mosart.r.0001-01-02-00000.nc.bp"
 
+# Convert ONLY coupler output in ADIOS BP format : Specify the regex for files to convert via "--bp-file" option
+jayesh@compute-386-03:~/scorpio_build2$ ./tools/adios2pio-nm/adios2pio-nm.exe --idir=/scratch/jayesh/e3sm/scratch/tmp_adios_bp_run_dir --bp-file=".*.cpl.*"
+WARNING: Skipping BP file (did not match specified regex): /scratch/jayesh/e3sm/scratch/tmp_adios_bp_run_dir/ERP_Ld3.ne4pg2_oQU480.F2010.anlgce-ub22_gnu.io-force_adios.20240717_210751_7we0ju.mpassi.rst.0001-01-03_00000.nc.bp
+WARNING: Skipping BP file (did not match specified regex): /scratch/jayesh/e3sm/scratch/tmp_adios_bp_run_dir/ERP_Ld3.ne4pg2_oQU480.F2010.anlgce-ub22_gnu.io-force_adios.20240717_210751_7we0ju.mosart.r.0001-01-03-00000.nc.bp
+WARNING: Skipping BP file (did not match specified regex): /scratch/jayesh/e3sm/scratch/tmp_adios_bp_run_dir/ERP_Ld3.ne4pg2_oQU480.F2010.anlgce-ub22_gnu.io-force_adios.20240717_210751_7we0ju.elm.r.0001-01-03-00000.nc.bp
+WARNING: Skipping BP file (did not match specified regex): /scratch/jayesh/e3sm/scratch/tmp_adios_bp_run_dir/ERP_Ld3.ne4pg2_oQU480.F2010.anlgce-ub22_gnu.io-force_adios.20240717_210751_7we0ju.eam.rh0.0001-01-03-00000.nc.bp
+WARNING: Skipping BP file (did not match specified regex): /scratch/jayesh/e3sm/scratch/tmp_adios_bp_run_dir/ERP_Ld3.ne4pg2_oQU480.F2010.anlgce-ub22_gnu.io-force_adios.20240717_210751_7we0ju.eam.r.0001-01-03-00000.nc.bp
+WARNING: Skipping BP file (did not match specified regex): /scratch/jayesh/e3sm/scratch/tmp_adios_bp_run_dir/ERP_Ld3.ne4pg2_oQU480.F2010.anlgce-ub22_gnu.io-force_adios.20240717_210751_7we0ju.mosart.rh0.0001-01-03-00000.nc.bp
+WARNING: Skipping BP file (did not match specified regex): /scratch/jayesh/e3sm/scratch/tmp_adios_bp_run_dir/ERP_Ld3.ne4pg2_oQU480.F2010.anlgce-ub22_gnu.io-force_adios.20240717_210751_7we0ju.elm.rh0.0001-01-03-00000.nc.bp
+WARNING: Skipping BP file (did not match specified regex): /scratch/jayesh/e3sm/scratch/tmp_adios_bp_run_dir/ERP_Ld3.ne4pg2_oQU480.F2010.anlgce-ub22_gnu.io-force_adios.20240717_210751_7we0ju.eam.rs.0001-01-03-00000.nc.bp
+Converting BP file : "/scratch/jayesh/e3sm/scratch/tmp_adios_bp_run_dir/ERP_Ld3.ne4pg2_oQU480.F2010.anlgce-ub22_gnu.io-force_adios.20240717_210751_7we0ju.cpl.hi.0001-01-04-00000.nc.bp"
+Converting BP file : "/scratch/jayesh/e3sm/scratch/tmp_adios_bp_run_dir/ERP_Ld3.ne4pg2_oQU480.F2010.anlgce-ub22_gnu.io-force_adios.20240717_210751_7we0ju.cpl.r.0001-01-03-00000.nc.bp"
+

tools/adios2pio-nm/adios2pio-nm-lib.cxx

@@ -2831,7 +2831,8 @@ static bool IsBPDir(const std::string &bp_dname,
  * corresponding file name prefixes to be used for converted
  * files
  */
-static int FindBPDirs(const string &bppdir,
+static int FindBPDirs(MPI_Comm comm, const string &bppdir,
+                      const string &bpdir_name_rgx,
                       vector<string> &bpdirs,
                       vector<string> &conv_fname_prefixes)
 {
@@ -2842,6 +2843,21 @@ static int FindBPDirs(const string &bppdir,
         return BP2PIO_ERROR;
     }
 
+    int rank = 0;
+    MPI_Comm_rank(comm, &rank);
+#ifdef SPIO_NO_CXX_REGEX
+    if(!bpdir_name_rgx.empty())
+    {
+        if(rank == 0)
+        {
+            printf("WARNING: C++ regex not supported, ignoring regex for converting files (converting all files)...\n");
+        }
+    }
+#else
+    std::smatch match;
+    std::regex rgx((!bpdir_name_rgx.empty()) ? bpdir_name_rgx : ".*");
+#endif
+
     struct dirent *pde = NULL;
     while ((pde = readdir(pdir)) != NULL)
     {
@@ -2852,6 +2868,16 @@ static int FindBPDirs(const string &bppdir,
         if ((pde->d_type == DT_DIR) &&
             IsBPDir(dname, dname_prefix))
         {
+#ifndef SPIO_NO_CXX_REGEX
+            if(!std::regex_match(dname, match, rgx))
+            {
+                if(rank == 0)
+                {
+                    printf("WARNING: Skipping BP file (did not match specified regex): %s\n", dname.c_str());
+                }
+                continue;
+            }
+#endif
             conv_fname_prefixes.push_back(dname_prefix);
             const std::string NC_SUFFIX(".nc");
             const std::string BP_SUFFIX(".bp");
@@ -2868,21 +2894,23 @@ static int FindBPDirs(const string &bppdir,
  * bppdir:  Directory containing multiple directories, named "*.bp",
  *          each directory containing BP files corresponding to a single
  *          file. This is the "BP Parent Directory".
+ * bpdir_name_rgx : Regex to match BP directories/files that need to be converted
  * piotype: The PIO IO type used for converting BP files to NetCDF using PIO
  * comm:    The MPI communicator to be used for conversion
  *
  * The function looks for all directories in bppdir named "*.bp"
  * and converts them, one at a time, to NetCDF files
  */
-int MConvertBPToNC(const string &bppdir, const string &piotype, const string &rearr,
+int MConvertBPToNC(const string &bppdir, const std::string &bpdir_name_rgx,
+                    const string &piotype, const string &rearr,
                     MPI_Comm comm, const std::string &rm_ifname_rgx)
 {
     int ierr = BP2PIO_NOERR;
     vector<string> bpdirs;
     vector<string> conv_fname_prefixes;
     const std::string CONV_FNAME_SUFFIX(".nc");
 
-    ierr = FindBPDirs(bppdir, bpdirs, conv_fname_prefixes);
+    ierr = FindBPDirs(comm, bppdir, bpdir_name_rgx, bpdirs, conv_fname_prefixes);
     if (ierr != BP2PIO_NOERR)
     {
         fprintf(stderr, "Unable to read directory, %s\n", bppdir.c_str());

tools/adios2pio-nm/adios2pio-nm-lib.h

@@ -19,8 +19,8 @@ int ConvertBPToNC(const string &infilepath,
                   const string &outfilename,
                   const string &piotype, const string &rearr, MPI_Comm comm_in,
                   const std::string &rm_ifname_rgx);
-int MConvertBPToNC(const string &bppdir, const string &piotype, const string &rearr,
-                    MPI_Comm comm, const std::string &rm_ifname_rgx);
+int MConvertBPToNC(const string &bppdir, const string &ifname_rgx, const string &piotype,
+                    const string &rearr, MPI_Comm comm, const std::string &rm_ifname_rgx);
 void SetDebugOutput(int val);
 
 #endif /* #ifndef _ADIOS2PIO_NM_LIB_H_ */

tools/adios2pio-nm/adios2pio-nm.cxx

@@ -9,10 +9,10 @@
 
 static void init_user_options(spio_tool_utils::ArgParser &ap)
 {
-  ap.add_opt("bp-file", "data produced by SCORPIO with ADIOS format")
-    .add_opt("rm-bp-file-after-conv", "remove the ADIOS BP file after conversion")
-    .add_opt("idir", "Directory containing data output from SCORPIO (in ADIOS format)")
-    .add_opt("nc-file", "output file name after conversion")
+  ap.add_opt("bp-file", "input file (or regex of basename for multiple files) in ADIOS BP format")
+    .add_opt("rm-bp-file-after-conv", "remove the ADIOS BP file after conversion (or regex to specify which files to delete after conversion)")
+    .add_opt("idir", "Directory containing the input files (in ADIOS BP format)")
+    .add_opt("nc-file", "output file name after conversion (will be used as prefix when converting multiple files)")
     .add_opt("pio-format", "output SCORPIO I/O type. Supported parameters: \"pnetcdf\",  \"netcdf\",  \"netcdf4c\",  \"netcdf4p\", \"nczarr\"")
     .add_opt("rearr", "SCORPIO rearranger. Supported parameters: \"subset\", \"box\", \"any\". Default \"any\".")
     .add_opt("verbose", "Turn on verbose info messages");
@@ -52,7 +52,7 @@ static int get_user_options(
               spio_tool_utils::ArgParser &ap,
               int argc, char *argv[],
               std::string &idir,
-              std::string &ifile, std::string &ofile,
+              std::string &ifname_rgx, std::string &ofile,
               std::string &rm_ifname_rgx,
               std::string &otype,
               std::string &rearr,
@@ -67,25 +67,20 @@ static int get_user_options(
 #else
   ap.parse(argc, argv);
 #endif
+
   if(!ap.has_arg("bp-file") && !ap.has_arg("idir")){
     ap.print_usage(std::cerr);
     return 1;
   }
+
   if(ap.has_arg("bp-file")){
-    ifile = ap.get_arg<std::string>("bp-file");
+    ifname_rgx = ap.get_arg<std::string>("bp-file");
     if(ap.has_arg("nc-file")){
       ofile = ap.get_arg<std::string>("nc-file");
     }
-    else{
-      ofile = strip_bp_ext(ifile);
-    }
-    if(ofile.size() == 0){
-      ap.print_usage(std::cerr);
-      return 1;
-    }
   }
-  else{
-    assert(ap.has_arg("idir"));
+
+  if(ap.has_arg("idir")){
     idir = ap.get_arg<std::string>("idir");
   }
 
@@ -145,11 +140,11 @@ int main(int argc, char *argv[])
   init_user_options(ap);
 
   /* Parse the user options */
-  string idir, infilepath, outfilename, piotype, rearr;
+  string idir, ifname_rgx, outfilename, piotype, rearr;
   string rm_ifname_rgx;
   int debug_lvl = 0;
   ret = get_user_options(ap, argc, argv,
-                          idir, infilepath, outfilename,
+                          idir, ifname_rgx, outfilename,
                           rm_ifname_rgx, piotype, rearr, debug_lvl, rank);
 
   if(ret != 0){
@@ -168,10 +163,11 @@ int main(int argc, char *argv[])
   SetDebugOutput(debug_lvl);
   MPI_Barrier(comm_in);
   if(idir.size() == 0){
-    ret = ConvertBPToNC(infilepath, outfilename, piotype, rearr, comm_in, rm_ifname_rgx);
+    /* FIXME: Hopefully ifname_rgx is actually a file name - not regex */
+    ret = ConvertBPToNC(ifname_rgx, outfilename, piotype, rearr, comm_in, rm_ifname_rgx);
   }
   else{
-    ret = MConvertBPToNC(idir, piotype, rearr, comm_in, rm_ifname_rgx);
+    ret = MConvertBPToNC(idir, ifname_rgx, piotype, rearr, comm_in, rm_ifname_rgx);
   }
   MPI_Barrier(comm_in);