Skip to content

Commit 79a593f

Browse files
jayeshkrishnajayeshkrishna
authored
Merge pull request #618 from E3SM-Project/jayeshkrishna/add_adiosc_iotype
Adding a new I/O type, PIO_IOTYPE_ADIOSC, that can be used to write (and read) data in ADIOS BP file format with data compression Also modifying the testing framework to test the new I/O type.
2 parents 9d8c58a + 1dab253 commit 79a593f

File tree

13 files changed

+190
-112
lines changed

13 files changed

+190
-112
lines changed

‎src/clib/pio.h‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1195,8 +1195,11 @@ enum PIO_IOTYPE
11951195
/** ADIOS parallel */
11961196
PIO_IOTYPE_ADIOS = 5,
11971197

1198+
/** ADIOS parallel with compression */
1199+
PIO_IOTYPE_ADIOSC = 6,
1200+
11981201
/** HDF5 parallel */
1199-
PIO_IOTYPE_HDF5 = 6
1202+
PIO_IOTYPE_HDF5 = 7
12001203
};
12011204

12021205
/**

‎src/clib/pio_darray.cpp‎

Lines changed: 13 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -2022,7 +2022,7 @@ int PIOc_write_darray_impl(int ncid, int varid, int ioid, PIO_Offset arraylen, c
20222022
spio_ltimer_start(ios->io_fstats->wr_timer_name);
20232023
spio_ltimer_start(ios->io_fstats->tot_timer_name);
20242024

2025-
if (file->iotype == PIO_IOTYPE_ADIOS)
2025+
if ((file->iotype == PIO_IOTYPE_ADIOS) || (file->iotype == PIO_IOTYPE_ADIOSC))
20262026
{
20272027
GPTLstart("PIO:PIOc_write_darray_adios");
20282028
GPTLstart("PIO:write_total_adios");
@@ -2043,7 +2043,7 @@ int PIOc_write_darray_impl(int ncid, int varid, int ioid, PIO_Offset arraylen, c
20432043
spio_ltimer_stop(ios->io_fstats->tot_timer_name);
20442044
spio_ltimer_stop(file->io_fstats->wr_timer_name);
20452045
spio_ltimer_stop(file->io_fstats->tot_timer_name);
2046-
if (file->iotype == PIO_IOTYPE_ADIOS)
2046+
if ((file->iotype == PIO_IOTYPE_ADIOS) || (file->iotype == PIO_IOTYPE_ADIOSC))
20472047
{
20482048
GPTLstop("PIO:PIOc_write_darray_adios");
20492049
GPTLstop("PIO:write_total_adios");
@@ -2061,7 +2061,7 @@ int PIOc_write_darray_impl(int ncid, int varid, int ioid, PIO_Offset arraylen, c
20612061
spio_ltimer_stop(ios->io_fstats->tot_timer_name);
20622062
spio_ltimer_stop(file->io_fstats->wr_timer_name);
20632063
spio_ltimer_stop(file->io_fstats->tot_timer_name);
2064-
if (file->iotype == PIO_IOTYPE_ADIOS)
2064+
if ((file->iotype == PIO_IOTYPE_ADIOS) || (file->iotype == PIO_IOTYPE_ADIOSC))
20652065
{
20662066
GPTLstop("PIO:PIOc_write_darray_adios");
20672067
GPTLstop("PIO:write_total_adios");
@@ -2082,7 +2082,7 @@ int PIOc_write_darray_impl(int ncid, int varid, int ioid, PIO_Offset arraylen, c
20822082
spio_ltimer_stop(ios->io_fstats->tot_timer_name);
20832083
spio_ltimer_stop(file->io_fstats->wr_timer_name);
20842084
spio_ltimer_stop(file->io_fstats->tot_timer_name);
2085-
if (file->iotype == PIO_IOTYPE_ADIOS)
2085+
if ((file->iotype == PIO_IOTYPE_ADIOS) || (file->iotype == PIO_IOTYPE_ADIOSC))
20862086
{
20872087
GPTLstop("PIO:PIOc_write_darray_adios");
20882088
GPTLstop("PIO:write_total_adios");
@@ -2114,7 +2114,7 @@ int PIOc_write_darray_impl(int ncid, int varid, int ioid, PIO_Offset arraylen, c
21142114
spio_ltimer_stop(ios->io_fstats->tot_timer_name);
21152115
spio_ltimer_stop(file->io_fstats->wr_timer_name);
21162116
spio_ltimer_stop(file->io_fstats->tot_timer_name);
2117-
if (file->iotype == PIO_IOTYPE_ADIOS)
2117+
if ((file->iotype == PIO_IOTYPE_ADIOS) || (file->iotype == PIO_IOTYPE_ADIOSC))
21182118
{
21192119
GPTLstop("PIO:PIOc_write_darray_adios");
21202120
GPTLstop("PIO:write_total_adios");
@@ -2191,7 +2191,7 @@ if (file->iotype != PIO_IOTYPE_HDF5)
21912191
{
21922192
GPTLstop("PIO:PIOc_write_darray");
21932193
GPTLstop("PIO:write_total");
2194-
if (file->iotype == PIO_IOTYPE_ADIOS)
2194+
if ((file->iotype == PIO_IOTYPE_ADIOS) || (file->iotype == PIO_IOTYPE_ADIOSC))
21952195
{
21962196
GPTLstop("PIO:PIOc_write_darray_adios");
21972197
GPTLstop("PIO:write_total_adios");
@@ -2219,7 +2219,7 @@ if (file->iotype != PIO_IOTYPE_HDF5)
22192219
LOG((3, "wmb->ioid = %d wmb->recordvar = %d", wmb->ioid, wmb->recordvar));
22202220

22212221
#ifdef _ADIOS2
2222-
if (file->iotype == PIO_IOTYPE_ADIOS)
2222+
if ((file->iotype == PIO_IOTYPE_ADIOS) || (file->iotype == PIO_IOTYPE_ADIOSC))
22232223
{
22242224
ierr = PIO_NOERR;
22252225
ierr = PIOc_write_darray_adios(file, varid, ioid, iodesc, arraylen, array, fillvalue);
@@ -3578,7 +3578,7 @@ int PIOc_read_darray_impl(int ncid, int varid, int ioid, PIO_Offset arraylen,
35783578
"unknown rearranger", __FILE__, __LINE__);
35793579

35803580
/* Get var description. */
3581-
if (file->iotype == PIO_IOTYPE_ADIOS)
3581+
if ((file->iotype == PIO_IOTYPE_ADIOS) || (file->iotype == PIO_IOTYPE_ADIOSC))
35823582
{
35833583
#ifdef _ADIOS2
35843584
vdesc_adios2 = &(file->adios_vars[varid]);
@@ -3597,7 +3597,7 @@ int PIOc_read_darray_impl(int ncid, int varid, int ioid, PIO_Offset arraylen,
35973597
spio_ltimer_stop(file->io_fstats->tot_timer_name);
35983598

35993599
/* Find out PIO data type of var. */
3600-
if (file->iotype == PIO_IOTYPE_ADIOS)
3600+
if ((file->iotype == PIO_IOTYPE_ADIOS) || (file->iotype == PIO_IOTYPE_ADIOSC))
36013601
{
36023602
#ifdef _ADIOS2
36033603
assert(vdesc_adios2->nc_type != PIO_NAT);
@@ -3620,7 +3620,7 @@ int PIOc_read_darray_impl(int ncid, int varid, int ioid, PIO_Offset arraylen,
36203620
}
36213621

36223622
/* Find out length of type. */
3623-
if (file->iotype == PIO_IOTYPE_ADIOS)
3623+
if ((file->iotype == PIO_IOTYPE_ADIOS) || (file->iotype == PIO_IOTYPE_ADIOSC))
36243624
{
36253625
#ifdef _ADIOS2
36263626
if (vdesc_adios2->adios_type_size == 0)
@@ -3649,7 +3649,7 @@ int PIOc_read_darray_impl(int ncid, int varid, int ioid, PIO_Offset arraylen,
36493649
spio_ltimer_start(file->io_fstats->tot_timer_name);
36503650
}
36513651

3652-
if (file->iotype == PIO_IOTYPE_ADIOS)
3652+
if ((file->iotype == PIO_IOTYPE_ADIOS) || (file->iotype == PIO_IOTYPE_ADIOSC))
36533653
{
36543654
#ifdef _ADIOS2
36553655
ios->io_fstats->rb += vdesc_adios2->adios_type_size * iodesc->llen;
@@ -3698,7 +3698,7 @@ int PIOc_read_darray_impl(int ncid, int varid, int ioid, PIO_Offset arraylen,
36983698
else
36993699
rlen = iodesc->llen;
37003700

3701-
if (file->iotype == PIO_IOTYPE_ADIOS)
3701+
if ((file->iotype == PIO_IOTYPE_ADIOS) || (file->iotype == PIO_IOTYPE_ADIOSC))
37023702
rlen = 0;
37033703

37043704
if(!ios->async || !ios->ioproc)
@@ -3822,6 +3822,7 @@ int PIOc_read_darray_impl(int ncid, int varid, int ioid, PIO_Offset arraylen,
38223822
break;
38233823
#ifdef _ADIOS2
38243824
case PIO_IOTYPE_ADIOS:
3825+
case PIO_IOTYPE_ADIOSC:
38253826
GPTLstart("PIO:PIOc_read_darray_adios");
38263827
if ((ierr = PIOc_read_darray_adios(file, fndims, iodesc, varid, array)))
38273828
{

‎src/clib/pio_file.cpp‎

Lines changed: 19 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,7 @@ int PIOc_createfile_impl(int iosysid, int *ncidp, const int *iotype, const char
135135

136136
GPTLstart("PIO:PIOc_createfile");
137137
GPTLstart("PIO:write_total");
138-
if (*iotype == PIO_IOTYPE_ADIOS)
138+
if ((*iotype == PIO_IOTYPE_ADIOS) || (*iotype == PIO_IOTYPE_ADIOSC))
139139
{
140140
GPTLstart("PIO:PIOc_createfile_adios");
141141
GPTLstart("PIO:write_total_adios");
@@ -146,7 +146,7 @@ int PIOc_createfile_impl(int iosysid, int *ncidp, const int *iotype, const char
146146
{
147147
GPTLstop("PIO:PIOc_createfile");
148148
GPTLstop("PIO:write_total");
149-
if (*iotype == PIO_IOTYPE_ADIOS)
149+
if ((*iotype == PIO_IOTYPE_ADIOS) || (*iotype == PIO_IOTYPE_ADIOSC))
150150
{
151151
GPTLstop("PIO:PIOc_createfile_adios");
152152
GPTLstop("PIO:write_total_adios");
@@ -165,7 +165,7 @@ int PIOc_createfile_impl(int iosysid, int *ncidp, const int *iotype, const char
165165
GPTLstop("PIO:write_total");
166166
spio_ltimer_stop(ios->io_fstats->wr_timer_name);
167167
spio_ltimer_stop(ios->io_fstats->tot_timer_name);
168-
if (*iotype == PIO_IOTYPE_ADIOS)
168+
if ((*iotype == PIO_IOTYPE_ADIOS) || (*iotype == PIO_IOTYPE_ADIOSC))
169169
{
170170
GPTLstop("PIO:PIOc_createfile_adios");
171171
GPTLstop("PIO:write_total_adios");
@@ -188,7 +188,7 @@ int PIOc_createfile_impl(int iosysid, int *ncidp, const int *iotype, const char
188188
{
189189
GPTLstop("PIO:PIOc_createfile");
190190
GPTLstop("PIO:write_total");
191-
if (*iotype == PIO_IOTYPE_ADIOS)
191+
if ((*iotype == PIO_IOTYPE_ADIOS) || (*iotype == PIO_IOTYPE_ADIOSC))
192192
{
193193
GPTLstop("PIO:PIOc_createfile_adios");
194194
GPTLstop("PIO:write_total_adios");
@@ -204,7 +204,7 @@ int PIOc_createfile_impl(int iosysid, int *ncidp, const int *iotype, const char
204204
GPTLstop("PIO:write_total");
205205
spio_ltimer_stop(ios->io_fstats->wr_timer_name);
206206
spio_ltimer_stop(ios->io_fstats->tot_timer_name);
207-
if (*iotype == PIO_IOTYPE_ADIOS)
207+
if ((*iotype == PIO_IOTYPE_ADIOS) || (*iotype == PIO_IOTYPE_ADIOSC))
208208
{
209209
GPTLstop("PIO:PIOc_createfile_adios");
210210
GPTLstop("PIO:write_total_adios");
@@ -290,7 +290,7 @@ static int sync_file(int ncid)
290290
spio_ltimer_start(file->io_fstats->tot_timer_name);
291291

292292
#ifdef _ADIOS2
293-
if (file->iotype == PIO_IOTYPE_ADIOS)
293+
if ((file->iotype == PIO_IOTYPE_ADIOS) || (file->iotype == PIO_IOTYPE_ADIOSC))
294294
{
295295
if (file->mode & PIO_WRITE)
296296
{
@@ -468,7 +468,7 @@ int PIOc_closefile_impl(int ncid)
468468
ios = file->iosystem;
469469
assert(ios);
470470

471-
if (file->iotype == PIO_IOTYPE_ADIOS)
471+
if ((file->iotype == PIO_IOTYPE_ADIOS) || (file->iotype == PIO_IOTYPE_ADIOSC))
472472
{
473473
GPTLstart("PIO:PIOc_closefile_adios");
474474

@@ -499,7 +499,7 @@ int PIOc_closefile_impl(int ncid)
499499
sync_file(ncid);
500500
}
501501

502-
if (file->iotype == PIO_IOTYPE_ADIOS)
502+
if ((file->iotype == PIO_IOTYPE_ADIOS) || (file->iotype == PIO_IOTYPE_ADIOSC))
503503
{
504504
#ifndef _ADIOS_BP2NC_TEST
505505
if (file->mode & PIO_WRITE)
@@ -533,7 +533,7 @@ int PIOc_closefile_impl(int ncid)
533533
PIO_SEND_ASYNC_MSG(ios, msg, &ierr, ncid);
534534
if(ierr != PIO_NOERR)
535535
{
536-
if (file->iotype == PIO_IOTYPE_ADIOS)
536+
if ((file->iotype == PIO_IOTYPE_ADIOS) || (file->iotype == PIO_IOTYPE_ADIOSC))
537537
{
538538
GPTLstop("PIO:PIOc_closefile_adios");
539539

@@ -572,7 +572,7 @@ int PIOc_closefile_impl(int ncid)
572572

573573
/* ADIOS: assume all procs are also IO tasks */
574574
#ifdef _ADIOS2
575-
if (file->iotype == PIO_IOTYPE_ADIOS)
575+
if ((file->iotype == PIO_IOTYPE_ADIOS) || (file->iotype == PIO_IOTYPE_ADIOSC))
576576
{
577577
adios2_error adiosErr = adios2_error_none;
578578

@@ -585,7 +585,7 @@ int PIOc_closefile_impl(int ncid)
585585
ierr = begin_adios2_step(file, NULL);
586586
if (ierr != PIO_NOERR)
587587
{
588-
if (file->iotype == PIO_IOTYPE_ADIOS)
588+
if ((file->iotype == PIO_IOTYPE_ADIOS) || (file->iotype == PIO_IOTYPE_ADIOSC))
589589
{
590590
GPTLstop("PIO:PIOc_closefile_adios");
591591
GPTLstop("PIO:write_total_adios");
@@ -619,7 +619,7 @@ int PIOc_closefile_impl(int ncid)
619619
attributeH = adios2_define_attribute(file->ioH, "/__pio__/fillmode", adios2_type_int32_t, &file->fillmode);
620620
if (attributeH == NULL)
621621
{
622-
if (file->iotype == PIO_IOTYPE_ADIOS)
622+
if ((file->iotype == PIO_IOTYPE_ADIOS) || (file->iotype == PIO_IOTYPE_ADIOSC))
623623
{
624624
GPTLstop("PIO:PIOc_closefile_adios");
625625
GPTLstop("PIO:write_total_adios");
@@ -680,7 +680,7 @@ int PIOc_closefile_impl(int ncid)
680680
GPTLstop("end_adios2_step_PIOc_closefile");
681681
if (ierr != PIO_NOERR)
682682
{
683-
if (file->iotype == PIO_IOTYPE_ADIOS)
683+
if ((file->iotype == PIO_IOTYPE_ADIOS) || (file->iotype == PIO_IOTYPE_ADIOSC))
684684
{
685685
GPTLstop("PIO:PIOc_closefile_adios");
686686
GPTLstop("PIO:write_total_adios");
@@ -709,7 +709,7 @@ int PIOc_closefile_impl(int ncid)
709709
adiosErr = adios2_close(file->engineH);
710710
if (adiosErr != adios2_error_none)
711711
{
712-
if (file->iotype == PIO_IOTYPE_ADIOS)
712+
if ((file->iotype == PIO_IOTYPE_ADIOS) || (file->iotype == PIO_IOTYPE_ADIOSC))
713713
{
714714
GPTLstop("PIO:PIOc_closefile_adios");
715715
GPTLstop("PIO:write_total_adios");
@@ -911,7 +911,7 @@ int PIOc_closefile_impl(int ncid)
911911
LOG((1, "DONE CONVERTING: %s", file->filename));
912912
if (ierr != PIO_NOERR)
913913
{
914-
if (file->iotype == PIO_IOTYPE_ADIOS)
914+
if ((file->iotype == PIO_IOTYPE_ADIOS) || (file->iotype == PIO_IOTYPE_ADIOSC))
915915
{
916916
GPTLstop("PIO:PIOc_closefile_adios");
917917
GPTLstop("PIO:write_total_adios");
@@ -941,7 +941,7 @@ int PIOc_closefile_impl(int ncid)
941941
file->filename = NULL;
942942
}
943943

944-
if (file->iotype == PIO_IOTYPE_ADIOS)
944+
if ((file->iotype == PIO_IOTYPE_ADIOS) || (file->iotype == PIO_IOTYPE_ADIOSC))
945945
{
946946
GPTLstop("PIO:PIOc_closefile_adios");
947947

@@ -983,7 +983,7 @@ int PIOc_closefile_impl(int ncid)
983983
}
984984
#endif
985985

986-
assert(file->iotype != PIO_IOTYPE_ADIOS);
986+
assert((file->iotype != PIO_IOTYPE_ADIOS) && (file->iotype != PIO_IOTYPE_ADIOSC));
987987

988988
/* If this is an IO task, then call the netCDF function. */
989989
if (ios->ioproc)
@@ -1243,7 +1243,7 @@ int PIOc_sync_impl(int ncid)
12431243
if (file->mode & PIO_WRITE)
12441244
{
12451245
GPTLstart("PIO:write_total");
1246-
if (file->iotype == PIO_IOTYPE_ADIOS)
1246+
if ((file->iotype == PIO_IOTYPE_ADIOS) || (file->iotype == PIO_IOTYPE_ADIOSC))
12471247
GPTLstart("PIO:write_total_adios");
12481248
}
12491249

@@ -1252,7 +1252,7 @@ int PIOc_sync_impl(int ncid)
12521252
if (file->mode & PIO_WRITE)
12531253
{
12541254
GPTLstop("PIO:write_total");
1255-
if (file->iotype == PIO_IOTYPE_ADIOS)
1255+
if ((file->iotype == PIO_IOTYPE_ADIOS) || (file->iotype == PIO_IOTYPE_ADIOSC))
12561256
GPTLstop("PIO:write_total_adios");
12571257
}
12581258

0 commit comments

Comments
 (0)