Skip to content

Commit a676327

Browse files
jayeshkrishnajayeshkrishna
authored
Merge pull request #660 from E3SM-Project/jayeshkrishna/hdf5_compression
Adding initial support for data compression using the HDF5 library. Also expanding the lossy compression options available with ADIOS (ZFP). A new I/O type, PIO_IOTYPE_HDF5C, is now available to write data in a compressed format using HDF5. Lossless compression using Blosc2 (Zstd) and lossy compression using ZFP is currently supported. The support also requires users to install the HDF5 filters for Blosc2 and ZFP. SCORPIO needs to be configured with -DHDF5_USE_COMPRESSION=ON to enable lossless data compression using HDF5 (Blosc2+Zstd) and adding -DHDF5_USE_LOSSY_COMPRESSION=ON switches the data compression to lossy compression using ZFP. Use the "-DSPIO_COMPRESSION_OPTIONS" configure option to specify detailed compression options for HDF5 and ADIOS2 libraries.
2 parents 16d80c8 + 1cc4ed2 commit a676327

29 files changed

+1529
-195
lines changed

‎CMakeLists.txt‎

Lines changed: 43 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ if (DEFINED USER_CMAKE_MODULE_PATH)
2323
CACHE STRING "Location of the CMake_Fortran_utils")
2424
list (APPEND CMAKE_MODULE_PATH ${USER_CMAKE_MODULE_PATH})
2525
endif ()
26+
include(SPIOCompressionUtils)
2627

2728
#==============================================================================
2829
# USER-DEFINED OPTIONS (set with "-DOPT=VAL" from command line)
@@ -54,10 +55,13 @@ option (WITH_NETCDF "Require the use of NetCDF" ON)
5455
option (PIO_ENABLE_NCZARR "Enable support for NCZarr (via NetCDF)" OFF)
5556
option (WITH_ADIOS2 "Require the use of ADIOS 2.x" OFF)
5657
option (WITH_HDF5 "Require the use of HDF5" OFF)
58+
option (SPIO_HDF5_FLUSH_AFTER_COLL_WR "Flush file buffers after each collective write" OFF)
5759
option (ADIOS_BP2NC_TEST "Enable testing of BP to NetCDF conversion" OFF)
5860
option (ADIOS_NO_DECOMPS "Save no decomposition data to ADIOS BP files" OFF)
5961
option (ADIOS_USE_COMPRESSION "Enable data compression methods in ADIOS (e.g., Blosc2, BZip2)" OFF)
6062
option (ADIOS_USE_LOSSY_COMPRESSION "Enable lossy compression methods in ADIOS (e.g., SZ, MGARD, ZFP)" OFF)
63+
option (HDF5_USE_COMPRESSION "Enable data compression methods in HDF5 (Lossless compression by default. e.g., Blosc2)" OFF)
64+
option (HDF5_USE_LOSSY_COMPRESSION "Enable lossy compression methods in HDF5 (e.g., ZFP)" OFF)
6165
#===== Testing Options =====
6266
option (PIO_ENABLE_TESTS "Enable the testing builds" OFF)
6367
option (PIO_ENABLE_LARGE_TESTS "Enable large (file, processes) tests" OFF)
@@ -269,6 +273,8 @@ else()
269273
message(STATUS "PIO_REARR_ANY rearranger : Setting the local decomposition map length range for SUBSET rearanger (PIO_REARR_ANY_SUBSET_RANGE) to " ${PIO_REARR_ANY_SUBSET_RANGE} " (default)")
270274
endif()
271275

276+
# A C++ regex pattern that matches nothing: \b\B (word boundary followed by non-word-boundary)
277+
set(CXX_REGEX_MATCHES_NOTHING "\\\\b\\\\B")
272278
if(WITH_ADIOS2)
273279
if(DEFINED SPIO_OVERRIDE_ADIOS_WITH_PNETCDF_FNAME_REGEX)
274280
if(WITH_PNETCDF)
@@ -277,13 +283,42 @@ if(WITH_ADIOS2)
277283
message(WARNING "Requested override of ADIOS with PnetCDF for file creation, but PnetCDF is not enabled. Override will be ignored.")
278284
endif()
279285
else()
280-
# Use a regex pattern that matches nothing: \b\B (word boundary followed by non-boundary)
286+
# Use a regex pattern that matches nothing
281287
# This effectively disables overriding ADIOS type with PnetCDF for file creation by default.
282-
set(SPIO_OVERRIDE_ADIOS_WITH_PNETCDF_FNAME_REGEX "\\b\\B")
288+
set(SPIO_OVERRIDE_ADIOS_WITH_PNETCDF_FNAME_REGEX ${CXX_REGEX_MATCHES_NOTHING})
283289
message(STATUS "Disabling override of ADIOS type with PnetCDF for file creation (default)")
284290
endif()
285291
endif()
286292

293+
if(SPIO_HDF5_FLUSH_AFTER_COLL_WR)
294+
set(SPIO_HDF5_FLUSH_AFTER_COLL_WR 0)
295+
if(WITH_HDF5)
296+
set(SPIO_HDF5_FLUSH_AFTER_COLL_WR 1)
297+
message(STATUS "Enabling flush after each collective write for HDF5 I/O type")
298+
endif()
299+
else()
300+
set(SPIO_HDF5_FLUSH_AFTER_COLL_WR 0)
301+
if(WITH_HDF5)
302+
message(STATUS "Disable flush after collective write for HDF5 I/O type (default)")
303+
endif()
304+
endif()
305+
306+
if(DEFINED SPIO_OVERRIDE_HDF5_COMPRESSION_VNAME_REGEX)
307+
if(WITH_HDF5 AND HDF5_USE_COMPRESSION)
308+
message(STATUS "Overriding HDF5 compression for variables matching regex: ${SPIO_OVERRIDE_HDF5_COMPRESSION_VNAME_REGEX}")
309+
endif()
310+
else()
311+
# Use a regex pattern that matches nothing, i.e., do NOT disable compression for any variable
312+
set(SPIO_OVERRIDE_ADIOS_WITH_PNETCDF_FNAME_REGEX ${CXX_REGEX_MATCHES_NOTHING})
313+
endif()
314+
315+
# Set the default compression options - HDF5/ADIOS + BLOSC2/ZFP
316+
spio_set_default_comp_opts()
317+
if(DEFINED SPIO_COMPRESSION_OPTIONS)
318+
message(STATUS "Parsing compression options")
319+
spio_parse_comp_opts("${SPIO_COMPRESSION_OPTIONS}")
320+
endif()
321+
287322
#==============================================================================
288323
# DETECT SYSTEM/COMPILERS (and set compiler specific options)
289324
#==============================================================================
@@ -316,6 +351,12 @@ if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
316351
if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS "4.9")
317352
message (WARNING "C++11 regex support is disabled since the compiler(" ${CMAKE_CXX_COMPILER_ID} ", version=" ${CMAKE_CXX_COMPILER_VERSION} ") does not support it")
318353
add_definitions(-DSPIO_NO_CXX_REGEX)
354+
if(WITH_ADIOS2 AND (DEFINED SPIO_OVERRIDE_ADIOS_WITH_PNETCDF_FNAME_REGEX))
355+
message(STATUS "C++ regex is not available. Overriding of ADIOS I/O type with PnetCDF for files will be disabled")
356+
endif()
357+
if(WITH_HDF5 AND (DEFINED SPIO_OVERRIDE_HDF5_COMPRESSION_VNAME_REGEX))
358+
message(STATUS "C++ regex is not available. Overriding HDF5 compression for variables (regex : ${SPIO_OVERRIDE_HDF5_COMPRESSION_VNAME_REGEX}) will be disabled")
359+
endif()
319360
endif()
320361
endif ()
321362

‎cmake/SPIOCompressionUtils.cmake‎

Lines changed: 299 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,299 @@
1+
#==============================================================================
2+
# HELPER MACROS
3+
#==============================================================================
4+
# Parse compression options
5+
# Call spio_parse_comp_opts() with the compression options to parse it
6+
# e.g. cmake -DSPIO_COMPRESSION_OPTIONS=
7+
# "COMPRESSION_LIBRARY=ADIOS2:Blosc2:Zstd;
8+
# SPIO_ADIOS2_BLOSC2_COMPRESSION_LEVEL=2;
9+
# SPIO_ADIOS2_BLOSC2_SHUFFLE_METHOD=BLOSC_BITSHUFFLE;
10+
# COMPRESSION_LIBRARY=HDF5:Blosc2:Zstd;
11+
# SPIO_HDF5_BLOSC2_COMPRESSION_LEVEL=2;
12+
# SPIO_HDF5_BLOSC2_SHUFFLE_METHOD=BLOSC_BITSHUFFLE;"
13+
14+
# Parse HDF5 + BLOSC2 compression options
15+
macro (spio_parse_hdf5_blosc2_comp_opts comp_opts)
16+
message(STATUS "Parsing HDF5:Blosc2 compression options")
17+
foreach(opt ${comp_opts})
18+
# Split each opt, OPTION_NAME=OPTION_VAL, to name & val
19+
string(REGEX MATCHALL "[^=]+" opt_args ${opt})
20+
list(LENGTH opt_args num_opt_args)
21+
if(num_opt_args EQUAL 2)
22+
list(GET opt_args 0 comp_lib_arg_name)
23+
list(GET opt_args 1 comp_lib_arg_val)
24+
if("${comp_lib_arg_name}" STREQUAL "SPIO_HDF5_BLOSC2_COMPRESSION_LEVEL")
25+
set(SPIO_HDF5_BLOSC2_COMPRESSION_LEVEL ${comp_lib_arg_val})
26+
elseif("${comp_lib_arg_name}" STREQUAL "SPIO_HDF5_BLOSC2_COMPRESSION_LIBRARY")
27+
set(SPIO_HDF5_BLOSC2_COMPRESSION_LIBRARY ${comp_lib_arg_val})
28+
elseif("${comp_lib_arg_name}" STREQUAL "SPIO_HDF5_BLOSC2_SHUFFLE_METHOD")
29+
set(SPIO_HDF5_BLOSC2_SHUFFLE_METHOD ${comp_lib_arg_val})
30+
elseif("${comp_lib_arg_name}" STREQUAL "COMPRESSION_LIBRARY")
31+
# The rest of the options are for another library, e.g. ADIOS options
32+
break()
33+
else()
34+
message(STATUS "ERROR: Invalid compression option (\"${opt}\"), cannot parse option, ignoring option")
35+
endif()
36+
else()
37+
message(STATUS "ERROR: Invalid compression option (\"${opt}\"), cannot parse option, ignoring option")
38+
endif()
39+
endforeach()
40+
endmacro()
41+
42+
# Parse HDF5 + ZFP compression options
43+
macro (spio_parse_hdf5_zfp_comp_opts comp_opts)
44+
message(STATUS "Parsing HDF5:zfp compression options")
45+
foreach(opt ${comp_opts})
46+
# Split each opt, OPTION_NAME=OPTION_VAL, to name & val
47+
string(REGEX MATCHALL "[^=]+" opt_args ${opt})
48+
list(LENGTH opt_args num_opt_args)
49+
if(num_opt_args EQUAL 2)
50+
list(GET opt_args 0 comp_lib_arg_name)
51+
list(GET opt_args 1 comp_lib_arg_val)
52+
if("${comp_lib_arg_name}" STREQUAL "SPIO_HDF5_ZFP_COMPRESSION_MODE")
53+
set(SPIO_HDF5_ZFP_COMPRESSION_MODE ${comp_lib_arg_val})
54+
elseif("${comp_lib_arg_name}" STREQUAL "SPIO_HDF5_ZFP_COMPRESSION_RATE")
55+
set(SPIO_HDF5_ZFP_COMPRESSION_RATE ${comp_lib_arg_val})
56+
elseif("${comp_lib_arg_name}" STREQUAL "SPIO_HDF5_ZFP_PRECISION")
57+
set(SPIO_HDF5_ZFP_PRECISION ${comp_lib_arg_val})
58+
elseif("${comp_lib_arg_name}" STREQUAL "SPIO_HDF5_ZFP_ACCURACY")
59+
set(SPIO_HDF5_ZFP_ACCURACY ${comp_lib_arg_val})
60+
elseif("${comp_lib_arg_name}" STREQUAL "COMPRESSION_LIBRARY")
61+
# The rest of the options are for another library, e.g. ADIOS options
62+
break()
63+
else()
64+
message(STATUS "ERROR: Invalid compression option (\"${opt}\"), cannot parse option, ignoring option")
65+
endif()
66+
else()
67+
message(STATUS "ERROR: Invalid compression option (\"${opt}\"), cannot parse option, ignoring option")
68+
endif()
69+
endforeach()
70+
endmacro()
71+
72+
# Parse all HDF5 compression options
73+
# e.g. spio_parse_hdf5_comp_opts("HDF5:BLOSC2:ZSTD", LIST_OF_COMPRESSION_OPTS)
74+
macro (spio_parse_hdf5_comp_opts comp_lib comp_opts)
75+
message(STATUS "Parsing HDF5 compression options")
76+
string(TOUPPER "${comp_lib}" comp_lib)
77+
string(STRIP "${comp_lib}" comp_lib)
78+
# Split compression library qualified name, "HDF5:BLOSC2:ZSTD" to individual lib names
79+
string(REGEX MATCHALL "[^:]+" comp_lib_detail ${comp_lib})
80+
# Expecting comp_lib = "HDF5:BLOSC2:ZSTD" OR "HDF5:ZFP"
81+
list(LENGTH comp_lib_detail num_comp_lib_detail)
82+
if(num_comp_lib_detail GREATER 1)
83+
list(GET comp_lib_detail 0 io_lib_name)
84+
list(GET comp_lib_detail 1 comp_lib_name)
85+
if("${comp_lib_name}" STREQUAL "ZFP")
86+
spio_parse_hdf5_zfp_comp_opts("${comp_opts}")
87+
elseif("${comp_lib_name}" STREQUAL "BLOSC2")
88+
if(num_comp_lib_detail GREATER 2)
89+
list(GET comp_lib_detail 2 comp_internal_lib_name)
90+
if("${comp_internal_lib_name}" STREQUAL "ZSTD")
91+
spio_parse_hdf5_blosc2_comp_opts("${comp_opts}")
92+
else()
93+
message(STATUS "ERROR: Invalid HDF5 compression option for BLOSC2 compression library. Internal compression library specified (${comp_internal_lib_name}) is not supported")
94+
endif()
95+
else()
96+
message(STATUS "ERROR: Invalid HDF5 compression option for BLOSC2 compression library. Internal compression library (e.g. ZSTD) was not specified")
97+
endif()
98+
else()
99+
message(STATUS "ERROR: Invalid HDF5 compression option, compression library specified (\"${comp_lib_name}\") is not supported")
100+
endif()
101+
else()
102+
message(STATUS "ERROR: Invalid HDF5 compression option, compression library specified (\"${comp_lib_detail}\") is not supported")
103+
endif()
104+
endmacro()
105+
106+
# Parse ADIOS2 + Blosc2 compression options
107+
macro (spio_parse_adios2_blosc2_comp_opts comp_opts)
108+
message(STATUS "Parsing ADIOS2:Blosc2 compression options")
109+
foreach(opt ${comp_opts})
110+
# Split each opt, OPTION_NAME=OPTION_VAL, to name & val
111+
string(REGEX MATCHALL "[^=]+" opt_args ${opt})
112+
list(LENGTH opt_args num_opt_args)
113+
if(num_opt_args EQUAL 2)
114+
list(GET opt_args 0 comp_lib_arg_name)
115+
list(GET opt_args 1 comp_lib_arg_val)
116+
if("${comp_lib_arg_name}" STREQUAL "SPIO_ADIOS2_BLOSC2_COMPRESSION_LEVEL")
117+
set(SPIO_ADIOS2_BLOSC2_COMPRESSION_LEVEL ${comp_lib_arg_val})
118+
elseif("${comp_lib_arg_name}" STREQUAL "SPIO_ADIOS2_BLOSC2_COMPRESSION_LIBRARY")
119+
set(SPIO_ADIOS2_BLOSC2_COMPRESSION_LIBRARY ${comp_lib_arg_val})
120+
elseif("${comp_lib_arg_name}" STREQUAL "SPIO_ADIOS2_BLOSC2_SHUFFLE_METHOD")
121+
set(SPIO_ADIOS2_BLOSC2_SHUFFLE_METHOD ${comp_lib_arg_val})
122+
elseif("${comp_lib_arg_name}" STREQUAL "COMPRESSION_LIBRARY")
123+
# The rest of the options are for another library, e.g. HDF5 options
124+
break()
125+
else()
126+
message(STATUS "ERROR: Invalid compression option (\"${opt}\"), cannot parse option, ignoring option")
127+
endif()
128+
else()
129+
message(STATUS "ERROR: Invalid compression option (\"${opt}\"), cannot parse option, ignoring option")
130+
endif()
131+
endforeach()
132+
endmacro()
133+
134+
# Parse ADIOS2 + ZFP compression options
135+
macro (spio_parse_adios2_zfp_comp_opts comp_opts)
136+
message(STATUS "Parsing ADIOS2:zfp compression options")
137+
foreach(opt ${comp_opts})
138+
# Split each opt, OPTION_NAME=OPTION_VAL, to name & val
139+
string(REGEX MATCHALL "[^=]+" opt_args ${opt})
140+
list(LENGTH opt_args num_opt_args)
141+
if(num_opt_args EQUAL 2)
142+
list(GET opt_args 0 comp_lib_arg_name)
143+
list(GET opt_args 1 comp_lib_arg_val)
144+
if("${comp_lib_arg_name}" STREQUAL "SPIO_ADIOS2_ZFP_COMPRESSION_MODE")
145+
set(SPIO_ADIOS2_ZFP_COMPRESSION_MODE ${comp_lib_arg_val})
146+
elseif("${comp_lib_arg_name}" STREQUAL "SPIO_ADIOS2_ZFP_COMPRESSION_RATE")
147+
set(SPIO_ADIOS2_ZFP_COMPRESSION_RATE ${comp_lib_arg_val})
148+
elseif("${comp_lib_arg_name}" STREQUAL "SPIO_ADIOS2_ZFP_PRECISION")
149+
set(SPIO_ADIOS2_ZFP_PRECISION ${comp_lib_arg_val})
150+
elseif("${comp_lib_arg_name}" STREQUAL "SPIO_ADIOS2_ZFP_ACCURACY")
151+
set(SPIO_ADIOS2_ZFP_ACCURACY ${comp_lib_arg_val})
152+
elseif("${comp_lib_arg_name}" STREQUAL "COMPRESSION_LIBRARY")
153+
# The rest of the options are for another library, e.g. HDF5 options
154+
break()
155+
else()
156+
message(STATUS "ERROR: Invalid compression option (\"${opt}\"), cannot parse option, ignoring option")
157+
endif()
158+
else()
159+
message(STATUS "ERROR: Invalid compression option (\"${opt}\"), cannot parse option, ignoring option")
160+
endif()
161+
endforeach()
162+
endmacro()
163+
164+
# Parse ADIOS2 + SZ compression options
165+
macro (spio_parse_adios2_sz_comp_opts comp_opts)
166+
message(STATUS "Parsing ADIOS2:SZ compression options")
167+
foreach(opt ${comp_opts})
168+
# Split each opt, OPTION_NAME=OPTION_VAL, to name & val
169+
string(REGEX MATCHALL "[^=]+" opt_args ${opt})
170+
list(LENGTH opt_args num_opt_args)
171+
if(num_opt_args EQUAL 2)
172+
list(GET opt_args 0 comp_lib_arg_name)
173+
list(GET opt_args 1 comp_lib_arg_val)
174+
if("${comp_lib_arg_name}" STREQUAL "SPIO_ADIOS2_SZ_ACCURACY")
175+
set(SPIO_ADIOS2_SZ_ACCURACY ${comp_lib_arg_val})
176+
elseif("${comp_lib_arg_name}" STREQUAL "COMPRESSION_LIBRARY")
177+
# The rest of the options are for another library, e.g. HDF5 options
178+
break()
179+
else()
180+
message(STATUS "ERROR: Invalid compression option (\"${opt}\"), cannot parse option, ignoring option")
181+
endif()
182+
else()
183+
message(STATUS "ERROR: Invalid compression option (\"${opt}\"), cannot parse option, ignoring option")
184+
endif()
185+
endforeach()
186+
endmacro()
187+
188+
# Parse all ADIOS2 compression options
189+
# e.g. spio_parse_adios2_comp_opts("ADIOS2:BLOSC2:ZSTD", LIST_OF_COMPRESSION_OPTS)
190+
macro (spio_parse_adios2_comp_opts comp_lib comp_opts)
191+
message(STATUS "Parsing ADIOS2 compression options")
192+
string(TOUPPER "${comp_lib}" comp_lib)
193+
string(STRIP "${comp_lib}" comp_lib)
194+
# Split compression library qualified name, "ADIOS2:BLOSC2:ZSTD" to individual lib names
195+
string(REGEX MATCHALL "[^:]+" comp_lib_detail ${comp_lib})
196+
# Expecting comp_lib = "ADIOS2:BLOSC2:ZSTD", "ADIOS2:SZ" or "ADIOS2:ZFP"
197+
list(LENGTH comp_lib_detail num_comp_lib_detail)
198+
if(num_comp_lib_detail GREATER 1)
199+
list(GET comp_lib_detail 0 io_lib_name)
200+
list(GET comp_lib_detail 1 comp_lib_name)
201+
if("${comp_lib_name}" STREQUAL "ZFP")
202+
spio_parse_adios2_zfp_comp_opts("${comp_opts}")
203+
elseif("${comp_lib_name}" STREQUAL "SZ")
204+
spio_parse_adios2_sz_comp_opts("${comp_opts}")
205+
elseif("${comp_lib_name}" STREQUAL "BLOSC2")
206+
if(num_comp_lib_detail GREATER 2)
207+
list(GET comp_lib_detail 2 comp_internal_lib_name)
208+
if("${comp_internal_lib_name}" STREQUAL "ZSTD")
209+
spio_parse_adios2_blosc2_comp_opts("${comp_opts}")
210+
else()
211+
message(STATUS "ERROR: Invalid ADIOS2 compression option for BLOSC2 compression library. Internal compression library specified (${comp_internal_lib_name}) is not supported")
212+
endif()
213+
else()
214+
message(STATUS "ERROR: Invalid ADIOS2 compression option for BLOSC2 compression library. Internal compression library (e.g. ZSTD) was not specified")
215+
endif()
216+
else()
217+
message(STATUS "ERROR: Invalid ADIOS2 compression option, compression library specified (\"${comp_lib_name}\") is not supported")
218+
endif()
219+
else()
220+
message(STATUS "ERROR: Invalid ADIOS2 compression option, compression library specified (\"${comp_lib_detail}\") is not supported")
221+
endif()
222+
endmacro()
223+
224+
macro (spio_set_default_comp_opts)
225+
# HDF5 + Blosc2
226+
set(SPIO_HDF5_BLOSC2_COMPRESSION_LEVEL 1)
227+
set(SPIO_HDF5_BLOSC2_COMPRESSION_LIBRARY "BLOSC_ZSTD")
228+
# BLOSC_NOSHUFFLE (0), BLOSC_SHUFFLE (1) : Byte shuffle,
229+
# BLOSC_BITSHUFFLE (2) : Bit shuffle
230+
set(SPIO_HDF5_BLOSC2_SHUFFLE_METHOD "BLOSC_BITSHUFFLE")
231+
232+
# HDF5 + ZFP : Available options are
233+
# "H5Z_ZFP_MODE_RATE" : Fixed bit rate (The number of bits for representing compressed data is fixed)
234+
# "H5Z_ZFP_MODE_PRECISION" : Fixed precision (Variable bit rate, The number of bits in original value is fixed)
235+
# "H5Z_ZFP_MODE_ACCURACY" : Fixed accuracy (Variable bit rate, The absolute error between compressed and original value is bound)
236+
# "H5Z_ZFP_MODE_REVERSIBLE" : Lossless compression
237+
set(SPIO_HDF5_ZFP_COMPRESSION_MODE "H5Z_ZFP_MODE_ACCURACY")
238+
set(SPIO_HDF5_ZFP_COMPRESSION_RATE 16)
239+
set(SPIO_HDF5_ZFP_PRECISION 16)
240+
set(SPIO_HDF5_ZFP_ACCURACY 0.001)
241+
242+
# ADIOS2 + Blosc2
243+
set(SPIO_ADIOS2_BLOSC2_COMPRESSION_LEVEL "1")
244+
set(SPIO_ADIOS2_BLOSC2_COMPRESSION_LIBRARY "zstd")
245+
set(SPIO_ADIOS2_BLOSC2_SHUFFLE_METHOD "BLOSC_BITSHUFFLE")
246+
247+
# ADIOS2 + ZFP
248+
# "ADIOS2_ZFP_MODE_RATE" : Fixed bit rate (no of bits is fixed in compressed value)
249+
# "ADIOS2_ZFP_MODE_PRECISION" : Fixed precision Variable bit rate (no of bits is fixed in original value)
250+
# "ADIOS2_ZFP_MODE_ACCURACY" : Fixed accuracy Variable bit rate (absolute error btw compressed and original value is less than upper bound)
251+
# "ADIOS2_ZFP_MODE_REVERSIBLE" (lossless)
252+
set(SPIO_ADIOS2_ZFP_COMPRESSION_MODE "ADIOS2_ZFP_MODE_ACCURACY")
253+
set(SPIO_ADIOS2_ZFP_COMPRESSION_RATE 16)
254+
set(SPIO_ADIOS2_ZFP_PRECISION 16)
255+
set(SPIO_ADIOS2_ZFP_ACCURACY "0.001")
256+
257+
# ADIOS2 + SZ
258+
set(SPIO_ADIOS2_SZ_ACCURACY "0.001")
259+
endmacro()
260+
261+
macro (spio_parse_comp_opts comp_opts)
262+
message(STATUS "Parsing compression options (-DSPIO_COMPRESSION_OPTIONS=\"${comp_opts}\")")
263+
set(copts)
264+
list(APPEND copts ${comp_opts})
265+
# Start idx for options after "COMPRESSION_LIBRARY=xxx"
266+
set(start_idx_opts 0)
267+
foreach(opt IN LISTS copts)
268+
# FIXME: Avoid reparsing options
269+
math(EXPR start_idx_opts "${start_idx_opts} + 1")
270+
string(REGEX MATCHALL "[^=]+" opt_args ${opt})
271+
list(LENGTH opt_args num_opt_args)
272+
if(num_opt_args EQUAL 2)
273+
list(GET opt_args 0 comp_lib_opt_name)
274+
list(GET opt_args 1 comp_lib)
275+
if(NOT ("${comp_lib_opt_name}" STREQUAL "COMPRESSION_LIBRARY"))
276+
# Skip other options - the rest of the options need to be parsed by library specific calls below
277+
continue()
278+
endif()
279+
string(TOUPPER "${comp_lib}" comp_lib)
280+
list(SUBLIST copts ${start_idx_opts} -1 copts_after_comp_lib_opt)
281+
# Check if compression library is hdf5
282+
string(FIND "${comp_lib}" "HDF5" pos)
283+
if(${pos} EQUAL 0)
284+
spio_parse_hdf5_comp_opts(${comp_lib} "${copts_after_comp_lib_opt}")
285+
else()
286+
# Check if compression library is adios
287+
string(FIND "${comp_lib}" "ADIOS2" pos)
288+
if(${pos} EQUAL 0)
289+
spio_parse_adios2_comp_opts(${comp_lib} "${copts_after_comp_lib_opt}")
290+
else()
291+
message(STATUS "ERROR: Invalid compression library (\"${comp_lib}\") specified in the compression option in options list (\"${comp_opts}\") in option \"${opt}\". Expected \"HDF5\" or \"ADIOS2\". Compression options will be ignored")
292+
break()
293+
endif()
294+
endif()
295+
else()
296+
message(STATUS "ERROR: Invalid compression option (\"${opt}\"), cannot parse option. Ignoring the option")
297+
endif()
298+
endforeach()
299+
endmacro()

0 commit comments

Comments
 (0)