Enable hint file, run the benchmark once per line in the hint file

hint file name is "e3sm_io_hints.txt"

Author: wkliao

e3sm_io_hints.txt

@@ -0,0 +1,2 @@
+start_iodevice=10;striping_unit=1048576;striping_factor=8;lustre_overstriping_ratio=1;cb_nodes=8
+start_iodevice=10;striping_unit=1048576;striping_factor=8;lustre_overstriping_ratio=1;cb_nodes=8;nc_num_aggrs_per_node=32

src/e3sm_io.c

@@ -76,21 +76,88 @@ void check_connector_env(e3sm_io_config *cfg) {
     free(env_str);
 }
 
-static inline int set_info (e3sm_io_config *cfg, e3sm_io_decom *decom) {
+static
+int parse_hint_line(e3sm_io_config *cfg,
+                    char           *hint_str)
+{
+    char *warn_str="Warning: skip ill-formed hint set in PNETCDF_HINTS";
+    char *next_hint, *key, *val, *deli;
+    int err=MPI_SUCCESS;
+
+    if (hint_str == NULL) return 1;
+
+    /* skip blank lines */
+    char *hint_saved = strdup(hint_str);
+    if (strlen(hint_str) == 0 || strtok(hint_saved, " \t") == NULL) {
+        free(hint_saved);
+        return 1;
+    }
+
+    next_hint = hint_str;
+    do {
+        deli = strchr(hint_str, ';');
+        if (deli != NULL) {
+            *deli = '\0'; /* add terminate char */
+            next_hint = deli + 1;
+        }
+        else break; /* done with this line */
+
+        /* skip all-blank hint, i.e. blanks in between 2 ';' */
+        if (strtok(NULL, " \t") == NULL) continue;
+
+        deli = strchr(hint_str, '=');
+        if (deli == NULL) { /* ill-formed hint */
+            printf("%s: '%s'\n", warn_str, hint_str);
+            continue;
+        }
+        *deli = '\0';
+
+        /* hint key */
+        key = strtok(NULL, "= \t");
+        if (key == NULL || NULL != strtok(NULL, "= \t")) {
+            /* expect one token before = */
+            printf("%s: '%s'\n", warn_str, hint_str);
+            continue;
+        }
+
+        /* hint value */
+        val = strtok(deli+1, "= \t");
+        if (NULL != strtok(NULL, "= \t")) {
+            /* expect one token before = */
+            printf("%s: '%s'\n", warn_str, hint_str);
+            continue;
+        }
+
+        /* override previouse set hint or add a new one */
+        err = MPI_Info_set(cfg->info, key, val);
+        CHECK_MPIERR
+
+    } while (*next_hint != '\0');
+
+err_out:
+    return err;
+}
+
+static inline int set_info(e3sm_io_config *cfg,
+                           char           *hint_str)
+{
     int err;
 
     /* set MPI-IO hints */
 
+    err = MPI_Info_create (&(cfg->info));
+    CHECK_MPIERR
+
     /* collective write */
     err = MPI_Info_set (cfg->info, "romio_cb_write", "enable");
     CHECK_MPIERR
 
     /* HDF5 may do independent I/O internally */
 
     if (cfg->api == pnetcdf) {
-        /* no independent MPI-IO */
-        err = MPI_Info_set (cfg->info, "romio_no_indep_rw", "true");
-        CHECK_MPIERR
+        /* set MPI-IO hints here */
+        // err = MPI_Info_set (cfg->info, "romio_no_indep_rw", "true");
+        // CHECK_MPIERR
 
         /* set PnetCDF I/O hints */
 
@@ -121,6 +188,9 @@ static inline int set_info (e3sm_io_config *cfg, e3sm_io_decom *decom) {
         /* in-place byte swap */
         err = MPI_Info_set (cfg->info, "nc_in_place_swap", "enable");
         CHECK_MPIERR
+
+        err = parse_hint_line(cfg, hint_str);
+        if (err == 0) goto err_out;
     }
 
 err_out:
@@ -224,7 +294,7 @@ int main (int argc, char **argv) {
     CHECK_ERR;
 #endif
 
-    timing[0] = MPI_Wtime();
+    timing[1] = MPI_Wtime();
 
     MPI_Comm_rank (MPI_COMM_WORLD, &(cfg.rank));
     MPI_Comm_size (MPI_COMM_WORLD, &(cfg.np));
@@ -562,7 +632,7 @@ int main (int argc, char **argv) {
     PRINT_MSG (1, "Input  data file/folder name = %s\n", cfg.in_path);
     PRINT_MSG (1, "Output data file/folder name = %s\n", cfg.out_path);
 
-    timing[1] = MPI_Wtime() - timing[0];
+    timing[1] = MPI_Wtime() - timing[1];
     MPI_Barrier(MPI_COMM_WORLD);
     timing[2] = MPI_Wtime();
 
@@ -584,41 +654,68 @@ int main (int argc, char **argv) {
         cfg.run_case = unknown;
 
     timing[2] = MPI_Wtime() - timing[2];
-    MPI_Barrier(MPI_COMM_WORLD);
-    timing[3] = MPI_Wtime();
 
-    /* set MPI-IO and PnetCDF hints */
-    err = MPI_Info_create (&(cfg.info));
-    CHECK_MPIERR
-    err = set_info (&cfg, &decom);
-    if (err < 0) goto err_out;
+    /* read I/O hints from file e3sm_io_hints.txt, run e3sm_io_core() one per
+     * line in the hint file. Each line contains hints as if it is set in the
+     * environment variable PNETCDF_HINTS.
+     */
+    FILE *fptr = fopen("e3sm_io_hints.txt", "r");
+    char hint_str[512];
+    int next_line;
+    if (fptr == NULL) /* ignore hint file if not exist */
+        next_line = EOF;
+    else /* get the first line of the hint file */
+        next_line = fscanf(fptr, "%s", hint_str);
 
-    /* the core of this benchmark */
-    err = e3sm_io_core (&cfg, &decom);
-    CHECK_ERR
+    do {
 
-    timing[3] = MPI_Wtime() - timing[3];
-    MPI_Barrier(MPI_COMM_WORLD);
-    timing[4] = MPI_Wtime();
+if (cfg.rank == 0) printf("\nHINTS: %s\n\n", (next_line == EOF) ? "" : hint_str);
 
-    /* report timing breakdowns */
-    if (cfg.rd) {
-        report_timing_RD(&cfg, &decom);
-    }
-    else{
-        report_timing_WR(&cfg, &decom);
-    }
+        /* the core of this benchmark */
+        MPI_Barrier(MPI_COMM_WORLD);
+        timing[3] = MPI_Wtime();
+
+        /* set MPI-IO and PnetCDF hints */
+        err = set_info(&cfg, (next_line == EOF) ?  NULL : hint_str);
+        if (err < 0) goto err_out;
+
+        err = e3sm_io_core(&cfg, &decom);
+        CHECK_ERR
+
+        timing[3] = MPI_Wtime() - timing[3];
+
+        /* report timing breakdowns */
+        if (cfg.rd)
+            report_timing_RD(&cfg, &decom);
+        else
+            report_timing_WR(&cfg, &decom);
 
 #ifdef E3SM_IO_PROFILING
-    if (cfg.profiling) e3sm_io_print_profile(&cfg);
+        if (cfg.profiling) e3sm_io_print_profile(&cfg);
 #else
-    if (cfg.profiling && cfg.rank == 0)
-        printf("\nWarning: E3SM-IO internal time profiling was disabled at configure time\n\n");
+        if (cfg.profiling && cfg.rank == 0)
+            printf("\nWarning: E3SM-IO internal time profiling was disabled at configure time\n\n");
 #endif
 
+        if (cfg.info != MPI_INFO_NULL) MPI_Info_free (&(cfg.info));
+
+        timing[0] = timing[1] + timing[2] + timing[3];
+        MPI_Reduce(timing, max_t, 4, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
+        if (cfg.rank == 0) {
+            printf("init=%.2f read_decomp=%.2f e3sm_io_core=%.2f end-to-end=%.2f\n",
+                   max_t[1],max_t[2],max_t[3],max_t[0]);
+            printf("-----------------------------------------------------------\n");
+            printf("\n\n");
+        }
+
+        if (fptr == NULL) break;
+
+        next_line = fscanf(fptr, "%s", hint_str);
+    } while (next_line != EOF);
+
+    if (fptr != NULL) fclose(fptr);
+
 err_out:
-    if (cfg.info != MPI_INFO_NULL)
-        MPI_Info_free (&(cfg.info));
     if (cfg.io_comm != MPI_COMM_WORLD && cfg.io_comm != MPI_COMM_NULL)
         MPI_Comm_free (&(cfg.io_comm));
     if (cfg.env_log_info != NULL)
@@ -635,16 +732,6 @@ int main (int argc, char **argv) {
         }
     }
 
-    timing[4] = MPI_Wtime() - timing[4];
-    timing[0] = MPI_Wtime() - timing[0];
-    MPI_Reduce(timing, max_t, 5, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
-    if (cfg.rank == 0) {
-        printf("init=%.2f read_decomp=%.2f e3sm_io_core=%.2f final=%.2f end-to-end=%.2f\n",
-               max_t[1],max_t[2],max_t[3],max_t[4],max_t[0]);
-        printf("-----------------------------------------------------------\n");
-        printf("\n\n");
-    }
-
     MPI_Finalize ();
 
     return (err < 0) ? 1 : 0;