Skip to content

Commit 91edd71

Browse files
dqwudqwu
authored
Merge pull request #648 from E3SM-Project/dqwu/fix_adios_append_check
Reopening files to append data is not supported by the ADIOS type. Returning an error when the user tries to append data to an existing file in ADIOS BP format. However, it is common for users to reopen existing files in write mode without performing any actual write operations, which is acceptable. An error will only be returned if an actual append operation is attempted on a reopened ADIOS file.
2 parents 78c22c2 + 3041870 commit 91edd71

File tree

4 files changed

+76
-18
lines changed

4 files changed

+76
-18
lines changed

‎src/clib/pio.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1193,6 +1193,9 @@ typedef struct file_desc_t
11931193
/** True if we need reserve some extra space in the header when
11941194
* creating NetCDF files to accommodate anticipated changes. */
11951195
bool reserve_extra_header_space;
1196+
1197+
/** True if this is an existing file reopened */
1198+
bool is_reopened;
11961199
} file_desc_t;
11971200

11981201
/**

‎src/clib/pio_darray.cpp‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2169,6 +2169,29 @@ int PIOc_write_darray_impl(int ncid, int varid, int ioid, PIO_Offset arraylen, c
21692169
"Writing variable (%s, varid=%d) to file (%s, ncid=%d) failed. The file was not opened for writing, try reopening the file in write mode (use the PIO_WRITE flag)", pio_get_vname_from_file(file, varid), varid, pio_get_fname_from_file(file), file->pio_ncid);
21702170
}
21712171

2172+
#ifdef _ADIOS2
2173+
if ((file->iotype == PIO_IOTYPE_ADIOS) || (file->iotype == PIO_IOTYPE_ADIOSC))
2174+
{
2175+
/* ADIOS type does not support open to append mode */
2176+
if (file->is_reopened)
2177+
{
2178+
GPTLstop("PIO:PIOc_write_darray");
2179+
GPTLstop("PIO:write_total");
2180+
GPTLstop("PIO:PIOc_write_darray_adios");
2181+
GPTLstop("PIO:write_total_adios");
2182+
spio_ltimer_stop(ios->io_fstats->wr_timer_name);
2183+
spio_ltimer_stop(ios->io_fstats->tot_timer_name);
2184+
spio_ltimer_stop(file->io_fstats->wr_timer_name);
2185+
spio_ltimer_stop(file->io_fstats->tot_timer_name);
2186+
return pio_err(ios, file, PIO_EADIOS2ERR, __FILE__, __LINE__,
2187+
"Writing variable (%s, varid=%d) to file (%s, ncid=%d) using ADIOS iotype failed. "
2188+
"Open to append mode is not supported yet",
2189+
pio_get_vname_from_file(file, varid), varid,
2190+
pio_get_fname_from_file(file), ncid);
2191+
}
2192+
}
2193+
#endif
2194+
21722195
/* Get decomposition information. */
21732196
if (!(iodesc = pio_get_iodesc_from_id(ioid)))
21742197
{

‎src/clib/pio_getput_int.cpp‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,27 @@ int spio_put_att_tc(int ncid, int varid, const char *name, nc_type atttype,
6060
spio_ltimer_start(ios->io_fstats->wr_timer_name);
6161
spio_ltimer_start(ios->io_fstats->tot_timer_name);
6262

63+
#ifdef _ADIOS2
64+
if ((file->iotype == PIO_IOTYPE_ADIOS) || (file->iotype == PIO_IOTYPE_ADIOSC))
65+
{
66+
/* ADIOS type does not support open to append mode */
67+
if (file->is_reopened)
68+
{
69+
GPTLstop("PIO:spio_put_att_tc");
70+
GPTLstop("PIO:write_total");
71+
spio_ltimer_stop(ios->io_fstats->wr_timer_name);
72+
spio_ltimer_stop(ios->io_fstats->tot_timer_name);
73+
spio_ltimer_stop(file->io_fstats->wr_timer_name);
74+
spio_ltimer_stop(file->io_fstats->tot_timer_name);
75+
return pio_err(ios, file, PIO_EADIOS2ERR, __FILE__, __LINE__,
76+
"Writing variable (%s, varid=%d) attribute (%s) to file (%s, ncid=%d) using ADIOS iotype failed. "
77+
"Open to append mode is not supported yet",
78+
pio_get_vname_from_file(file, varid), varid, PIO_IS_NULL(name),
79+
pio_get_fname_from_file(file), ncid);
80+
}
81+
}
82+
#endif
83+
6384
if ((file->iotype == PIO_IOTYPE_ADIOS) || (file->iotype == PIO_IOTYPE_ADIOSC))
6485
{
6586
GPTLstart("PIO:spio_put_att_tc_adios");
@@ -2463,6 +2484,27 @@ int spio_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off
24632484
spio_ltimer_start(ios->io_fstats->wr_timer_name);
24642485
spio_ltimer_start(ios->io_fstats->tot_timer_name);
24652486

2487+
#ifdef _ADIOS2
2488+
if ((file->iotype == PIO_IOTYPE_ADIOS) || (file->iotype == PIO_IOTYPE_ADIOSC))
2489+
{
2490+
/* ADIOS type does not support open to append mode */
2491+
if (file->is_reopened)
2492+
{
2493+
GPTLstop("PIO:spio_put_vars_tc");
2494+
GPTLstop("PIO:write_total");
2495+
spio_ltimer_stop(ios->io_fstats->wr_timer_name);
2496+
spio_ltimer_stop(ios->io_fstats->tot_timer_name);
2497+
spio_ltimer_stop(file->io_fstats->wr_timer_name);
2498+
spio_ltimer_stop(file->io_fstats->tot_timer_name);
2499+
return pio_err(ios, file, PIO_EADIOS2ERR, __FILE__, __LINE__,
2500+
"Writing variable (%s, varid=%d) to file (%s, ncid=%d) using ADIOS iotype failed. "
2501+
"Open to append mode is not supported yet",
2502+
pio_get_vname_from_file(file, varid), varid,
2503+
pio_get_fname_from_file(file), ncid);
2504+
}
2505+
}
2506+
#endif
2507+
24662508
if ((file->iotype == PIO_IOTYPE_ADIOS) || (file->iotype == PIO_IOTYPE_ADIOSC))
24672509
{
24682510
GPTLstart("PIO:spio_put_vars_tc_adios");

‎src/clib/pioc_support.cpp‎

Lines changed: 8 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -3045,6 +3045,7 @@ int spio_createfile_int(int iosysid, int *ncidp, const int *iotype, const char *
30453045
/* Fill in some file values. */
30463046
file->fh = -1;
30473047
file->reserve_extra_header_space = true; /* Set to true for creating output NetCDF files only. */
3048+
file->is_reopened = false;
30483049
strncpy(file->fname, filename, PIO_MAX_NAME);
30493050
ierr = pio_create_uniq_str(ios, NULL, tname, SPIO_TIMER_MAX_NAME, "tmp_", "_file");
30503051
if(ierr != PIO_NOERR)
@@ -4704,6 +4705,7 @@ int PIOc_openfile_retry_impl(int iosysid, int *ncidp, int *iotype, const char *f
47044705
/* Fill in some file values. */
47054706
file->fh = -1;
47064707
file->reserve_extra_header_space = false; /* Set to true for creating output NetCDF files only. */
4708+
file->is_reopened = true;
47074709
strncpy(file->fname, filename, PIO_MAX_NAME);
47084710
ierr = pio_create_uniq_str(ios, NULL, tname, SPIO_TIMER_MAX_NAME, "tmp_", "_file");
47094711
if(ierr != PIO_NOERR)
@@ -4725,18 +4727,6 @@ int PIOc_openfile_retry_impl(int iosysid, int *ncidp, int *iotype, const char *f
47254727
#ifdef _ADIOS2
47264728
if ((file->iotype == PIO_IOTYPE_ADIOS) || (file->iotype == PIO_IOTYPE_ADIOSC))
47274729
{
4728-
if (file->mode & PIO_WRITE)
4729-
{
4730-
spio_ltimer_stop(ios->io_fstats->rd_timer_name);
4731-
spio_ltimer_stop(ios->io_fstats->tot_timer_name);
4732-
spio_ltimer_stop(file->io_fstats->rd_timer_name);
4733-
spio_ltimer_stop(file->io_fstats->tot_timer_name);
4734-
return pio_err(ios, file, PIO_EADIOS2ERR, __FILE__, __LINE__,
4735-
"Opening file (%s) using ADIOS iotype failed. "
4736-
"Open to append mode is not supported yet",
4737-
filename);
4738-
}
4739-
47404730
/* Trying to open a file with adios unless ADIOS_BP2NC_TEST option is enabled for unit tests */
47414731
bool adios2_file_exist = false;
47424732

@@ -4762,7 +4752,7 @@ int PIOc_openfile_retry_impl(int iosysid, int *ncidp, int *iotype, const char *f
47624752
spio_ltimer_stop(ios->io_fstats->tot_timer_name);
47634753
spio_ltimer_stop(file->io_fstats->rd_timer_name);
47644754
spio_ltimer_stop(file->io_fstats->tot_timer_name);
4765-
return pio_err(ios, file, PIO_EADIOS2ERR, __FILE__, __LINE__,
4755+
return pio_err(ios, NULL, PIO_EADIOS2ERR, __FILE__, __LINE__,
47664756
"Opening file (%s) using ADIOS iotype failed. "
47674757
"The low level (ADIOS) I/O library call failed to declare a new io handler",
47684758
filename);
@@ -4775,7 +4765,7 @@ int PIOc_openfile_retry_impl(int iosysid, int *ncidp, int *iotype, const char *f
47754765
spio_ltimer_stop(ios->io_fstats->tot_timer_name);
47764766
spio_ltimer_stop(file->io_fstats->rd_timer_name);
47774767
spio_ltimer_stop(file->io_fstats->tot_timer_name);
4778-
return pio_err(ios, file, PIO_EADIOS2ERR, __FILE__, __LINE__,
4768+
return pio_err(ios, NULL, PIO_EADIOS2ERR, __FILE__, __LINE__,
47794769
"Opening file (%s) using ADIOS iotype failed. "
47804770
"The low level (ADIOS) I/O library call failed to set a single parameter (adios2_error=%s)",
47814771
filename, convert_adios2_error_to_string(adiosErr));
@@ -4788,7 +4778,7 @@ int PIOc_openfile_retry_impl(int iosysid, int *ncidp, int *iotype, const char *f
47884778
spio_ltimer_stop(ios->io_fstats->tot_timer_name);
47894779
spio_ltimer_stop(file->io_fstats->rd_timer_name);
47904780
spio_ltimer_stop(file->io_fstats->tot_timer_name);
4791-
return pio_err(ios, file, PIO_EADIOS2ERR, __FILE__, __LINE__,
4781+
return pio_err(ios, NULL, PIO_EADIOS2ERR, __FILE__, __LINE__,
47924782
"Opening file (%s) using ADIOS iotype failed. "
47934783
"The low level (ADIOS) I/O library call failed to set a single parameter (adios2_error=%s)",
47944784
filename, convert_adios2_error_to_string(adiosErr));
@@ -4801,7 +4791,7 @@ int PIOc_openfile_retry_impl(int iosysid, int *ncidp, int *iotype, const char *f
48014791
spio_ltimer_stop(ios->io_fstats->tot_timer_name);
48024792
spio_ltimer_stop(file->io_fstats->rd_timer_name);
48034793
spio_ltimer_stop(file->io_fstats->tot_timer_name);
4804-
return pio_err(ios, file, PIO_EADIOS2ERR, __FILE__, __LINE__,
4794+
return pio_err(ios, NULL, PIO_EADIOS2ERR, __FILE__, __LINE__,
48054795
"Opening file (%s) using ADIOS iotype failed. "
48064796
"The low level (ADIOS) I/O library call failed to set a single parameter (adios2_error=%s)",
48074797
filename, convert_adios2_error_to_string(adiosErr));
@@ -4814,7 +4804,7 @@ int PIOc_openfile_retry_impl(int iosysid, int *ncidp, int *iotype, const char *f
48144804
spio_ltimer_stop(ios->io_fstats->tot_timer_name);
48154805
spio_ltimer_stop(file->io_fstats->rd_timer_name);
48164806
spio_ltimer_stop(file->io_fstats->tot_timer_name);
4817-
return pio_err(ios, file, PIO_EADIOS2ERR, __FILE__, __LINE__,
4807+
return pio_err(ios, NULL, PIO_EADIOS2ERR, __FILE__, __LINE__,
48184808
"Opening file (%s) using ADIOS iotype failed. "
48194809
"The low level (ADIOS) I/O library call failed to set a single parameter (adios2_error=%s)",
48204810
filename, convert_adios2_error_to_string(adiosErr));
@@ -4827,7 +4817,7 @@ int PIOc_openfile_retry_impl(int iosysid, int *ncidp, int *iotype, const char *f
48274817
spio_ltimer_stop(ios->io_fstats->tot_timer_name);
48284818
spio_ltimer_stop(file->io_fstats->rd_timer_name);
48294819
spio_ltimer_stop(file->io_fstats->tot_timer_name);
4830-
return pio_err(ios, file, PIO_EADIOS2ERR, __FILE__, __LINE__,
4820+
return pio_err(ios, NULL, PIO_EADIOS2ERR, __FILE__, __LINE__,
48314821
"Opening file (%s) using ADIOS iotype failed. "
48324822
"The low level (ADIOS) I/O library call failed to set the engine type for current io handler (adios2_error=%s)",
48334823
filename, convert_adios2_error_to_string(adiosErr));

0 commit comments

Comments
 (0)