Skip to content

Commit 074611e

Browse files
dqwudqwu
authored
Merge pull request #630 from E3SM-Project/dqwu/adios_lossy_compression
This PR adds lossy compression methods (e.g., SZ, MGARD, ZFP) to further reduce the size of the output ADIOS BP files. SZ is set as the default lossy compression method due to its performance advantages over other methods.
2 parents ebd2a7d + 30b9a17 commit 074611e

File tree

6 files changed

+201
-50
lines changed

6 files changed

+201
-50
lines changed

‎CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ option (WITH_HDF5 "Require the use of HDF5" OFF)
5555
option (ADIOS_BP2NC_TEST "Enable testing of BP to NetCDF conversion" OFF)
5656
option (ADIOS_NO_DECOMPS "Save no decomposition data to ADIOS BP files" OFF)
5757
option (ADIOS_USE_COMPRESSION "Enable data compression methods in ADIOS (e.g., Blosc2, BZip2)" OFF)
58+
option (ADIOS_USE_LOSSY_COMPRESSION "Enable lossy compression methods in ADIOS (e.g., SZ, MGARD, ZFP)" OFF)
5859
#===== Testing Options =====
5960
option (PIO_ENABLE_TESTS "Enable the testing builds" OFF)
6061
option (PIO_ENABLE_LARGE_TESTS "Enable large (file, processes) tests" OFF)

‎ctest/CTestEnvironment-anlgce.cmake‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,11 @@ if (DEFINED ENV{ENABLE_ADIOS_USE_COMPRESSION})
9696
set (CTEST_CONFIGURE_OPTIONS "${CTEST_CONFIGURE_OPTIONS} -DADIOS_USE_COMPRESSION=ON")
9797
endif ()
9898

99+
# If ENABLE_ADIOS_USE_LOSSY_COMPRESSION environment variable is set, then enable lossy compression methods in ADIOS (e.g., SZ, MGARD, ZFP)
100+
if (DEFINED ENV{ENABLE_ADIOS_USE_LOSSY_COMPRESSION})
101+
set (CTEST_CONFIGURE_OPTIONS "${CTEST_CONFIGURE_OPTIONS} -DADIOS_USE_LOSSY_COMPRESSION=ON")
102+
endif ()
103+
99104
# If ENABLE_TESTS environment variable is set, then enable the testing builds
100105
if (DEFINED ENV{ENABLE_TESTS})
101106
set (CTEST_CONFIGURE_OPTIONS "${CTEST_CONFIGURE_OPTIONS} -DPIO_ENABLE_TESTS=ON")

‎src/clib/CMakeLists.txt‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -324,11 +324,26 @@ if (ADIOS_NO_DECOMPS)
324324
endif ()
325325

326326
if (ADIOS_USE_COMPRESSION)
327-
message(STATUS "Applying data compression to ADIOS BP files to reduce disk usage")
327+
message(STATUS "Applying data compression to ADIOS BP files")
328+
message(STATUS "Currently supported lossless compression methods (ADIOS operators): Blosc2 (default), BZip2")
329+
message(STATUS "Blosc2 internal compressor (algorithm: blosclz, lz4, lz4hc, snappy, zlib, zstd) = zstd (default)")
330+
message(STATUS "Blosc2 compression level (range: 0 – 9) = 1 (default)")
331+
message(STATUS "Blosc2 shuffle mode (option: BLOSC_SHUFFLE, BLOSC_NOSHUFFLE, BLOSC_BITSHUFFLE) = BLOSC_BITSHUFFLE (default)")
332+
message(STATUS "These are the default settings for lossless compression. Future versions will support configuration or run-time options")
328333
target_compile_definitions (pioc
329334
PUBLIC _SPIO_ADIOS_USE_COMPRESSION)
330335
endif ()
331336

337+
if (ADIOS_USE_LOSSY_COMPRESSION)
338+
message(STATUS "Applying lossy compression to ADIOS BP files")
339+
message(STATUS "Currently supported lossy compression methods (ADIOS operators): SZ (default), MGARD, ZFP")
340+
message(STATUS "Lossy compression error bound = 0.001 (default)")
341+
message(STATUS "These are the default settings for lossy compression. Future versions will support configuration or run-time options")
342+
message(WARNING "The SZ or MGARD methods only support float or double types. For other types, lossless compression will be used instead")
343+
target_compile_definitions (pioc
344+
PRIVATE _SPIO_ADIOS_USE_LOSSY_COMPRESSION)
345+
endif ()
346+
332347
if (PIO_ENABLE_API_TRACING)
333348
message(STATUS "Enabling API tracing")
334349
set(ENABLE_API_TRACING 1)

‎src/clib/pio.h‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -774,8 +774,14 @@ typedef struct iosystem_desc_t
774774
MPI_Comm block_comm;
775775
int block_myrank, block_nprocs;
776776
#ifdef _SPIO_ADIOS_USE_COMPRESSION
777-
/* ADIOS operator for applying a specific compression method (e.g., Blosc2, BZip2) */
778-
adios2_operator* compression_operator;
777+
/* ADIOS operator for applying a specific lossless compression method (e.g., Blosc2, BZip2) */
778+
adios2_operator* lossless_compression_operator;
779+
int adios_lossless_compression_method;
780+
#ifdef _SPIO_ADIOS_USE_LOSSY_COMPRESSION
781+
/* ADIOS operator for applying a specific lossy compression method (e.g., SZ, MGARD, ZFP) */
782+
adios2_operator* lossy_compression_operator;
783+
int adios_lossy_compression_method;
784+
#endif
779785
#endif
780786
#endif
781787

@@ -914,6 +920,21 @@ typedef struct adios_att_desc_t
914920
/** Type converted from NC type to adios type */
915921
adios2_type adios_type;
916922
} adios_att_desc_t;
923+
924+
#ifdef _SPIO_ADIOS_USE_COMPRESSION
925+
enum ADIOS_COMPRESSION_METHOD
926+
{
927+
ADIOS_COMPRESSION_METHOD_BLOSC2 = 1,
928+
929+
ADIOS_COMPRESSION_METHOD_BZIP2 = 2,
930+
931+
ADIOS_COMPRESSION_METHOD_MGARD = 3,
932+
933+
ADIOS_COMPRESSION_METHOD_SZ = 4,
934+
935+
ADIOS_COMPRESSION_METHOD_ZFP = 5
936+
};
937+
#endif
917938
#endif /* _ADIOS2 */
918939

919940
#ifdef _HDF5

‎src/clib/pioc.cpp‎

Lines changed: 59 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1374,26 +1374,66 @@ int PIOc_Init_Intracomm_impl(MPI_Comm comp_comm, int num_iotasks, int stride, in
13741374
return pio_err(ios, NULL, PIO_EADIOS2ERR, __FILE__, __LINE__, "Initializing ADIOS failed");
13751375
}
13761376

1377-
#ifdef _SPIO_ADIOS_USE_COMPRESSION
1377+
#ifdef _SPIO_ADIOS_USE_COMPRESSION
1378+
ios->lossless_compression_operator = NULL;
1379+
1380+
/* FIXME: Get lossless compression method during run-time */
1381+
ios->adios_lossless_compression_method = ADIOS_COMPRESSION_METHOD_BLOSC2; /* Use BLOSC2 as the default lossless compression method */
1382+
13781383
/* adios2_define_operator defines an adios2 supported operator by its type. Returns: handler (success) or NULL (failure) */
1379-
#if defined(ADIOS2_HAVE_BLOSC2)
1380-
/* Prefer Blosc2 for data compression due to its performance benefits over BZip2 */
1381-
ios->compression_operator = adios2_define_operator(ios->adiosH, "Blosc2Lossless", "blosc");
1382-
#elif defined(ADIOS2_HAVE_BZIP2)
1383-
/* Fall back to BZip2 if Blosc2 is not available */
1384-
ios->compression_operator = adios2_define_operator(ios->adiosH, "BZip2Lossless", "bzip2");
1385-
#else
1386-
/* No supported compression operators are available */
1387-
ios->compression_operator = NULL;
1388-
#endif
1389-
1390-
/* Check if a valid compression operator was defined; if not, warn the user */
1391-
if (ios->compression_operator == NULL)
1392-
{
1393-
printf("PIO: WARNING: Failed to define an adios2 supported compression operator (e.g., Blosc2, BZip2), "
1394-
"or no supported compression operators are available. Data compression will be disabled.\n");
1395-
}
1396-
#endif
1384+
if (ios->adios_lossless_compression_method == ADIOS_COMPRESSION_METHOD_BLOSC2)
1385+
{
1386+
#ifdef ADIOS2_HAVE_BLOSC2
1387+
ios->lossless_compression_operator = adios2_define_operator(ios->adiosH, "Blosc2Lossless", "blosc");
1388+
#endif
1389+
}
1390+
else if (ios->adios_lossless_compression_method == ADIOS_COMPRESSION_METHOD_BZIP2)
1391+
{
1392+
#ifdef ADIOS2_HAVE_BZIP2
1393+
ios->lossless_compression_operator = adios2_define_operator(ios->adiosH, "BZip2Lossless", "bzip2");
1394+
#endif
1395+
}
1396+
1397+
/* Check if a valid lossless compression operator was defined; if not, warn the user */
1398+
if (ios->lossless_compression_operator == NULL)
1399+
{
1400+
printf("PIO: WARNING: Failed to define an adios2 supported lossless compression operator (e.g., Blosc2, BZip2), "
1401+
"or no supported lossless compression operators are available. Lossless compression will be disabled.\n");
1402+
}
1403+
1404+
#ifdef _SPIO_ADIOS_USE_LOSSY_COMPRESSION
1405+
ios->lossy_compression_operator = NULL;
1406+
1407+
/* FIXME: Get lossy compression method during run-time */
1408+
ios->adios_lossy_compression_method = ADIOS_COMPRESSION_METHOD_SZ; /* Use SZ as the default lossy compression method */
1409+
1410+
if (ios->adios_lossy_compression_method == ADIOS_COMPRESSION_METHOD_MGARD)
1411+
{
1412+
#ifdef ADIOS2_HAVE_MGARD
1413+
ios->lossy_compression_operator = adios2_define_operator(ios->adiosH, "MGARDLossy", "mgard");
1414+
#endif
1415+
}
1416+
if (ios->adios_lossy_compression_method == ADIOS_COMPRESSION_METHOD_SZ)
1417+
{
1418+
#ifdef ADIOS2_HAVE_SZ
1419+
ios->lossy_compression_operator = adios2_define_operator(ios->adiosH, "SZLossy", "sz");
1420+
#endif
1421+
}
1422+
else if (ios->adios_lossy_compression_method == ADIOS_COMPRESSION_METHOD_ZFP)
1423+
{
1424+
#ifdef ADIOS2_HAVE_ZFP
1425+
ios->lossy_compression_operator = adios2_define_operator(ios->adiosH, "ZFPLossy", "zfp");
1426+
#endif
1427+
}
1428+
1429+
/* Check if a valid lossy compression operator was defined; if not, warn the user */
1430+
if (ios->lossy_compression_operator == NULL)
1431+
{
1432+
printf("PIO: WARNING: Failed to define an adios2 supported lossy compression operator (e.g., SZ, MGARD, ZFP), "
1433+
"or no supported lossy compression operators are available. Lossy compression will be disabled.\n");
1434+
}
1435+
#endif /* _SPIO_ADIOS_USE_LOSSY_COMPRESSION */
1436+
#endif /* _SPIO_ADIOS_USE_COMPRESSION */
13971437
}
13981438

13991439
ios->adios_readerH = adios2_init_mpi(ios->union_comm);

‎src/clib/pioc_support.cpp‎

Lines changed: 97 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -553,46 +553,115 @@ adios2_variable* spio_define_adios2_variable(iosystem_desc_t *ios, file_desc_t *
553553
#ifdef _SPIO_ADIOS_USE_COMPRESSION
554554
assert(ios != NULL && file != NULL);
555555
/* Skip compression for scalar variables (ndims == 0), see https://github.com/ornladios/ADIOS2/issues/4390 */
556-
if (variable != NULL && ios->compression_operator != NULL && ndims > 0 && (file->iotype == PIO_IOTYPE_ADIOSC))
556+
if (variable != NULL && ndims > 0 && (file->iotype == PIO_IOTYPE_ADIOSC))
557557
{
558558
size_t operation_index = 0;
559559
adios2_error adiosErr = adios2_error_none;
560560

561-
#if defined(ADIOS2_HAVE_BLOSC2)
562-
/* Add a compression operation (Blosc2) to this variable */
563-
adiosErr = adios2_add_operation(&operation_index, variable, ios->compression_operator, "compressor", "zstd"); /* Compressor type (e.g., zstd) */
564-
if (adiosErr != adios2_error_none)
561+
/* Use lossless compression method by default */
562+
int variable_compression_method = ios->adios_lossless_compression_method;
563+
564+
#ifdef _SPIO_ADIOS_USE_LOSSY_COMPRESSION
565+
/* Switch to lossy compression method */
566+
variable_compression_method = ios->adios_lossy_compression_method;
567+
568+
/* SZ or MGARD compressor only supports float or double types */
569+
if (variable_compression_method == ADIOS_COMPRESSION_METHOD_SZ || variable_compression_method == ADIOS_COMPRESSION_METHOD_MGARD)
565570
{
566-
pio_err(ios, file, PIO_EADIOS2ERR, __FILE__, __LINE__,
567-
"Failed to add Blosc2 compression operation (compressor=zstd) to variable %s (adios2_error=%s)",
568-
name, convert_adios2_error_to_string(adiosErr));
571+
/* Switch back to lossless compression method for non floating point type */
572+
if (type != adios2_type_float && type != adios2_type_double)
573+
variable_compression_method = ios->adios_lossless_compression_method;
569574
}
575+
#endif
570576

571-
/* Compression level is an integer between 0 (no compression) and 9 (more compression, more memory consumption) inclusive */
572-
adiosErr = adios2_set_operation_parameter(variable, operation_index, "clevel", "5");
573-
if (adiosErr != adios2_error_none)
577+
if (variable_compression_method == ADIOS_COMPRESSION_METHOD_BLOSC2)
574578
{
575-
pio_err(ios, file, PIO_EADIOS2ERR, __FILE__, __LINE__,
576-
"Failed to set Blosc2 compression level (clevel=5) for variable %s (adios2_error=%s)",
577-
name, convert_adios2_error_to_string(adiosErr));
578-
}
579+
if (ios->lossless_compression_operator != NULL)
580+
{
581+
/* Add a compression operation (Blosc2) to this variable */
582+
adiosErr = adios2_add_operation(&operation_index, variable, ios->lossless_compression_operator, "compressor", "zstd"); /* Compressor type (e.g., zstd) */
583+
if (adiosErr != adios2_error_none)
584+
{
585+
pio_err(ios, file, PIO_EADIOS2ERR, __FILE__, __LINE__,
586+
"Failed to add Blosc2 compression operation (compressor=zstd) to variable %s (adios2_error=%s)",
587+
name, convert_adios2_error_to_string(adiosErr));
588+
}
579589

580-
/* Shuffle option (BLOSC_SHUFFLE/BLOSC_NOSHUFFLE/BLOSC_BITSHUFFLE) */
581-
adiosErr = adios2_set_operation_parameter(variable, operation_index, "doshuffle", "BLOSC_BITSHUFFLE");
582-
if (adiosErr != adios2_error_none)
590+
/* Compression level is an integer between 0 (no compression) and 9 (more compression, more memory consumption) inclusive */
591+
adiosErr = adios2_set_operation_parameter(variable, operation_index, "clevel", "1");
592+
if (adiosErr != adios2_error_none)
593+
{
594+
pio_err(ios, file, PIO_EADIOS2ERR, __FILE__, __LINE__,
595+
"Failed to set Blosc2 compression level (clevel=5) for variable %s (adios2_error=%s)",
596+
name, convert_adios2_error_to_string(adiosErr));
597+
}
598+
599+
/* Shuffle option (BLOSC_SHUFFLE/BLOSC_NOSHUFFLE/BLOSC_BITSHUFFLE) */
600+
adiosErr = adios2_set_operation_parameter(variable, operation_index, "doshuffle", "BLOSC_BITSHUFFLE");
601+
if (adiosErr != adios2_error_none)
602+
{
603+
pio_err(ios, file, PIO_EADIOS2ERR, __FILE__, __LINE__,
604+
"Failed to set Blosc2 shuffle (doshuffle=BLOSC_BITSHUFFLE) for variable %s (adios2_error=%s)",
605+
name, convert_adios2_error_to_string(adiosErr));
606+
}
607+
}
608+
}
609+
else if (variable_compression_method == ADIOS_COMPRESSION_METHOD_BZIP2)
583610
{
584-
pio_err(ios, file, PIO_EADIOS2ERR, __FILE__, __LINE__,
585-
"Failed to set Blosc2 shuffle (doshuffle=BLOSC_BITSHUFFLE) for variable %s (adios2_error=%s)",
586-
name, convert_adios2_error_to_string(adiosErr));
611+
if (ios->lossless_compression_operator != NULL)
612+
{
613+
/* Add a compression operation (BZip2) to this variable */
614+
adiosErr = adios2_add_operation(&operation_index, variable, ios->lossless_compression_operator, "", "");
615+
if (adiosErr != adios2_error_none)
616+
{
617+
pio_err(ios, file, PIO_EADIOS2ERR, __FILE__, __LINE__,
618+
"Failed to add BZip2 compression operation to variable %s (adios2_error=%s)",
619+
name, convert_adios2_error_to_string(adiosErr));
620+
}
621+
}
587622
}
588-
#elif defined(ADIOS2_HAVE_BZIP2)
589-
/* Add a compression operation (BZip2) to this variable */
590-
adiosErr = adios2_add_operation(&operation_index, variable, ios->compression_operator, "", "");
591-
if (adiosErr != adios2_error_none)
623+
#ifdef _SPIO_ADIOS_USE_LOSSY_COMPRESSION
624+
else if (variable_compression_method == ADIOS_COMPRESSION_METHOD_MGARD)
625+
{
626+
if (ios->lossy_compression_operator != NULL)
627+
{
628+
/* Add a lossy compression operation (MGARD) to this variable */
629+
adiosErr = adios2_add_operation(&operation_index, variable, ios->lossy_compression_operator, "accuracy", "0.001");
630+
if (adiosErr != adios2_error_none)
631+
{
632+
pio_err(ios, file, PIO_EADIOS2ERR, __FILE__, __LINE__,
633+
"Failed to add MGARD compression operation to variable %s (adios2_error=%s)",
634+
name, convert_adios2_error_to_string(adiosErr));
635+
}
636+
}
637+
}
638+
else if (variable_compression_method == ADIOS_COMPRESSION_METHOD_SZ)
639+
{
640+
if (ios->lossy_compression_operator != NULL)
641+
{
642+
/* Add a lossy compression operation (SZ) to this variable */
643+
adiosErr = adios2_add_operation(&operation_index, variable, ios->lossy_compression_operator, "accuracy", "0.001");
644+
if (adiosErr != adios2_error_none)
645+
{
646+
pio_err(ios, file, PIO_EADIOS2ERR, __FILE__, __LINE__,
647+
"Failed to add SZ compression operation to variable %s (adios2_error=%s)",
648+
name, convert_adios2_error_to_string(adiosErr));
649+
}
650+
}
651+
}
652+
else if (variable_compression_method == ADIOS_COMPRESSION_METHOD_ZFP)
592653
{
593-
pio_err(ios, file, PIO_EADIOS2ERR, __FILE__, __LINE__,
594-
"Failed to add BZip2 compression operation to variable %s (adios2_error=%s)",
595-
name, convert_adios2_error_to_string(adiosErr));
654+
if (ios->lossy_compression_operator != NULL)
655+
{
656+
/* Add a lossy compression operation (ZFP) to this variable */
657+
adiosErr = adios2_add_operation(&operation_index, variable, ios->lossy_compression_operator, "accuracy", "0.001");
658+
if (adiosErr != adios2_error_none)
659+
{
660+
pio_err(ios, file, PIO_EADIOS2ERR, __FILE__, __LINE__,
661+
"Failed to add ZFP compression operation to variable %s (adios2_error=%s)",
662+
name, convert_adios2_error_to_string(adiosErr));
663+
}
664+
}
596665
}
597666
#endif
598667
}

0 commit comments

Comments
 (0)