Adding a configure option for HDF5/PnetCDF chunked variables

For HDF5 type, record variables are currently chunked using the
full dimension length. With very large dimensions, this can cause
dataset creation to fail in the HDF5 library.

Introduce a configure option to set the chunk size (in bytes) for
HDF5/PnetCDF chunked variables, with a default of 4 MB.

Author: dqwu

CMakeLists.txt

@@ -229,6 +229,20 @@ else()
   message(STATUS "Reserving some extra space in the header when creating NetCDF files, requested bytes = " ${PIO_RESERVED_FILE_HEADER_SIZE} " (default)")
 endif()
 
+# Set chunk size (in bytes) for HDF5/PnetCDF chunked variables. Default is 4 MB, same as NetCDF4.
+set(DEF_SPIO_CHUNK_SIZE 4194304)
+if(DEFINED PIO_CHUNK_SIZE)
+  if(PIO_CHUNK_SIZE GREATER_EQUAL 1)
+    message(STATUS "Setting chunk size (in bytes) for HDF5/PnetCDF chunked variables, requested bytes = " ${PIO_CHUNK_SIZE})
+  else()
+    message(WARNING "User-defined PIO_CHUNK_SIZE is invalid, setting it to " ${DEF_SPIO_CHUNK_SIZE} " (default)")
+    set(PIO_CHUNK_SIZE ${DEF_SPIO_CHUNK_SIZE})
+  endif()
+else()
+  set(PIO_CHUNK_SIZE ${DEF_SPIO_CHUNK_SIZE})
+  message(STATUS "Setting chunk size (in bytes) for HDF5/PnetCDF chunked variables, requested bytes = " ${PIO_CHUNK_SIZE} " (default)")
+endif()
+
 if(WITH_ADIOS2)
   # Maximum number of I/O decompositions registered with ADIOS type
   set(DEF_SPIO_MAX_ADIOS_DECOMPS 65536)

src/clib/pio_config.h.in

@@ -55,6 +55,9 @@
 /** Extra bytes reserved in the header when creating NetCDF files. */
 #define PIO_RESERVED_FILE_HEADER_SIZE @PIO_RESERVED_FILE_HEADER_SIZE@
 
+/** Set chunk size (in bytes) for HDF5/PnetCDF chunked variables. */
+#define PIO_CHUNK_SIZE @PIO_CHUNK_SIZE@
+
 /** Maximum number of I/O decompositions registered with ADIOS type */
 #define PIO_MAX_ADIOS_DECOMPS @PIO_MAX_ADIOS_DECOMPS@