Add chunking and compression driver

Author: khou2020

configure.ac

@@ -141,6 +141,9 @@ AH_TEMPLATE([RELAX_COORD_BOUND],        [Define if relaxed coordinate check is e
 AH_TEMPLATE([ENABLE_LARGE_SINGLE_REQ],  [Define if to enable large single MPI-IO request])
 AH_TEMPLATE([ENABLE_NULL_BYTE_HEADER_PADDING], [Define if to enable strict null-byte padding in file header])
 AH_TEMPLATE([ENABLE_BURST_BUFFER],      [Define if to enable burst buffer feature])
+AH_TEMPLATE([ENABLE_COMPRESSION],       [Define if to enable compression feature])
+AH_TEMPLATE([ENABLE_ZLIB],              [Define if to enable zlib compression method])
+AH_TEMPLATE([ENABLE_SZ],                [Define if to enable sz compression method])
 AH_TEMPLATE([PNETCDF_PROFILING],        [Define if to enable PnetCDF internal performance profiling])
 AH_TEMPLATE([ENABLE_THREAD_SAFE],       [Define if to enable thread-safe capability])
 AH_TEMPLATE([ENABLE_REQ_AGGREGATION],   [Define if able to support request aggregation in nonblocking routines])
@@ -2210,6 +2213,129 @@ fi
 AC_SUBST(ENABLE_BURST_BUFFER)
 AM_CONDITIONAL(ENABLE_BURST_BUFFER, [test x$enable_bbdriver = xyes])
 
+AC_ARG_ENABLE([compression],
+   [AS_HELP_STRING([--enable-compression],
+                   [Enable chunked compression driver support. @<:@default: disabled@:>@])],
+   [enable_compression=${enableval}], [enable_compression=no]
+)
+
+ENABLE_COMPRESSION=0
+if test "x$enable_compression" = "xyes" ; then
+   AC_DEFINE(ENABLE_COMPRESSION)
+   ENABLE_COMPRESSION=1
+fi
+AC_SUBST(ENABLE_COMPRESSION)
+AM_CONDITIONAL(ENABLE_COMPRESSION, [test x$enable_compression = xyes])
+
+AC_ARG_ENABLE([zlib],
+   [AS_HELP_STRING([--enable-zlib],
+                   [Enable zlib compression method support. @<:@default: disabled@:>@])],
+   [enable_zlib=${enableval}], [enable_zlib=no]
+)
+
+ENABLE_ZLIB=0
+if test "x$enable_zlib" = "xyes" ; then
+   AC_DEFINE(ENABLE_ZLIB)
+   ENABLE_ZLIB=1
+fi
+AC_SUBST(ENABLE_ZLIB)
+AM_CONDITIONAL(ENABLE_ZLIB, [test x$enable_zlib = xyes])
+
+if test "x$enable_zlib" = "xyes" ; then
+   ZLIB_INSTALL=""
+   AC_ARG_WITH(zlib,
+       [AS_HELP_STRING([--with-zlib=/path/to/implementation],
+                       [installation prefix for zlib implementation])],
+       if test "x${withval}" = xyes; then
+          AC_MSG_ERROR(--with-zlib is set but the value is NULL)
+       else
+          ZLIB_INSTALL=${withval}
+       fi
+   )
+   
+   if test "x${ZLIB_INSTALL}" != x ; then
+      CPPFLAGS+=" -I${ZLIB_INSTALL}/include"
+      LDFLAGS+=" -L${ZLIB_INSTALL}/lib"
+      LIBS+=" -lz"
+   fi
+
+   LIBS+=" -lm -ldl"
+
+   have_zlib=no
+   AC_MSG_CHECKING(ZLIB library)
+   AC_SEARCH_LIBS([deflate], [z], [have_zlib=yes], [have_zlib=no])
+   if test "x${have_zlib}" = xyes; then
+      AC_CHECK_HEADERS([zlib.h], [], [have_zlib=no])
+   fi
+
+   if test "x${have_zlib}" = xno; then
+      AC_MSG_ERROR([
+      ------------------------------------------------------------
+      The ZLIB library and header file are required to build
+      PnetCDF with ZLIB compression support. Use option
+         --with-zlib=/path/to/implementation
+      to specify the location of ZLIB build.
+      Stopping ...
+      Check 'config.log' for more information.
+      ------------------------------------------------------------])
+   fi
+fi
+
+AC_ARG_ENABLE([sz],
+   [AS_HELP_STRING([--enable-sz],
+                   [Enable sz compression method support. @<:@default: disabled@:>@])],
+   [enable_sz=${enableval}], [enable_sz=no]
+)
+
+ENABLE_SZ=0
+if test "x$enable_sz" = "xyes" ; then
+   AC_DEFINE(ENABLE_SZ)
+   ENABLE_SZ=1
+fi
+AC_SUBST(ENABLE_SZ)
+AM_CONDITIONAL(ENABLE_SZ, [test x$enable_sz = xyes])
+
+
+if test "x$enable_sz" = "xyes" ; then
+   SZ_INSTALL=""
+   AC_ARG_WITH(sz,
+       [AS_HELP_STRING([--with-sz=/path/to/implementation],
+                       [installation prefix for sz implementation])],
+       if test "x${withval}" = xyes; then
+          AC_MSG_ERROR(--with-sz is set but the value is NULL)
+       else
+          SZ_INSTALL=${withval}
+       fi
+   )
+   
+   if test "x${SZ_INSTALL}" != x ; then
+      CPPFLAGS+=" -I${SZ_INSTALL}/include"
+      LDFLAGS+=" -L${SZ_INSTALL}/lib"
+      LIBS+=" -lSZ -lzstd"
+   fi
+
+   LIBS+=" -lm -ldl"
+
+   have_sz=no
+   AC_MSG_CHECKING(SZ library)
+   AC_SEARCH_LIBS([deflate], [z], [have_sz=yes], [have_sz=no])
+   if test "x${have_sz}" = xyes; then
+      AC_CHECK_HEADERS([sz.h], [], [have_sz=no])
+   fi
+
+   if test "x${have_sz}" = xno; then
+      AC_MSG_ERROR([
+      ------------------------------------------------------------
+      The SZ library and header file are required to build
+      PnetCDF with SZ compression support. Use option
+         --with-sz=/path/to/implementation
+      to specify the location of SZ build.
+      Stopping ...
+      Check 'config.log' for more information.
+      ------------------------------------------------------------])
+   fi
+fi
+
 ADIOS_INSTALL=""
 AC_ARG_WITH(adios,
    [AS_HELP_STRING([--with-adios@<:@=DIR@:>@],

src/dispatchers/file.c

@@ -438,6 +438,11 @@ ncmpi_create(MPI_Comm    comm,
     if (enable_bb_driver)
         driver = ncbbio_inq_driver();
     else
+#endif
+#ifdef ENABLE_COMPRESSION
+    if (enable_zip_driver)
+        driver = nczipio_inq_driver();
+    else
 #endif
         /* default is the driver built on top of MPI-IO */
         driver = ncmpio_inq_driver();
@@ -692,6 +697,11 @@ ncmpi_open(MPI_Comm    comm,
     if (enable_bb_driver)
         driver = ncbbio_inq_driver();
     else
+#endif
+#ifdef ENABLE_COMPRESSION
+    if (enable_zip_driver)
+        driver = nczipio_inq_driver();
+    else
 #endif
     {
         /* ncmpio driver */

src/drivers/Makefile.am

@@ -24,6 +24,10 @@ if ENABLE_ADIOS
    SUBDIRS += ncadios
 endif
 
+if ENABLE_COMPRESSION
+   SUBDIRS += nczipio
+endif
+
 DIST_SUBDIRS = include common ncmpio ncfoo ncbbio nc4io ncadios
 
 # For VPATH build (parallel build), try delete all sub-directories

src/drivers/nczipio/DEVELOPER_NOTES.md

@@ -0,0 +1,74 @@
+# Note for Developers
+
+### Table of contents
+- [Future Work]
+- [Internal global attributes]
+- [Anchor variable (one per variable with chunking enabled)]
+- [Reference table]
+- [Chunks]
+- [Requirement for compressed variables]
+
+---
+
+## Internal global attributes:
+  * Number of chunked variables
+
+## Anchor variable (one per variable with chunking enabled):
+  * A scalar variable
+  * Data type is the same as user defined
+  * Internal attributes
+    + Dimension IDs are saved as an attribute of an array of integer type
+    + Number of dimensions is saved as an internal attribute
+    + An attribute to tell whether it is a fixed-size or record variable
+    + An attribute offset pointer to reference table
+      * For fixed-size variable, it is a scalar
+      * For record variable, it is an array of 8-type integers, one for each record
+        * This array can be allocated in multiple of 16 for example
+        * Need an integer for allocated size, e.g. multiple of 16
+        * Need an integer for size (true number of records written)
+    + An attributes for chunk sizes, an integer array
+    + An attributes for compression algorithm
+    + An attributes for compression level
+  * If a variable missing these internal attributes, it is a traditional variable
+
+## Reference table:
+  * An array stores offsets of individual chunks
+  * Not a NetCDF variable. But we use the CDF5 format specification to define it
+    + TODO: give it a formal spec in BNF grammar
+  * For a fixed-size variable, it is a 1D array of size equal to the number of chunks
+  * This table is loaded into memory when calling ncmpi_inq_varid
+  * For blocking API, it is sync-ed and written to file by root
+    + TODO: in future, it can be written by multiple ranks in parallel
+  * For nonblocking API, multiple tables are written by multiple ranks in parallel
+
+## Chunks:
+  * Chunks are not NetCDF variables
+    + TODO: give it a formal spec in BNF grammar?
+  * Chunks are stored in space between NetCDF variables, i.e. padding areas in files
+  * Data is type-converted and byte-swapped before compression
+  * In principle, chunks should be stored in file contiguously with each other,
+    for all variables. But they are not required to be stored contiguously.
+  * The storage order of chunks is in row major
+
+## Requirement for compressed variables:
+  * Collective I/O only (this is the same required by HDF5)
+  * Must be chunked (same as HDF5)
+
+
+## Future Work
+* Reuse metadata accross variables
+  - Variable from same simulation space may have same access apttern.
+  - Instead of generating variable metadata and indexx table separately, we can 
+    share information accross variables.
+  - Chunk sizeand chunk ownership info can be reused.
+* Data seiving
+  - When rewriting to a chunk, we do't need to read the background if it is 
+    fully overwritten.
+  - Need an efficient way to determine whether a chunk is fully rewrititen. 
+  - It may be infesible due to communication and computation cost.
+  - HDF5 approximate this by checking if owner fully rewriten the chunk.
+* Reuse metadata accross records
+  - I/O pattern accross time steps are likely the same.
+  - If we detect same I/O pattern as previous record, we can skip sending the metadata. 
+  - MPI datatype created for previous timestep can also be reused. 
+---

src/drivers/nczipio/Makefile.am

@@ -0,0 +1,97 @@
+#
+# Copyright (C) 2012, Northwestern University and Argonne National Laboratory
+# See COPYRIGHT notice in top-level directory.
+#
+# $Id: Makefile.am 3283 2017-07-30 21:10:11Z wkliao $
+#
+# @configure_input@
+
+SUFFIXES = .a .o .c .m4 .h
+
+AM_CPPFLAGS  = -I${top_srcdir}/src/include
+AM_CPPFLAGS += -I${top_builddir}/src/include
+AM_CPPFLAGS += -I${top_srcdir}/src/drivers/include
+AM_CPPFLAGS += -I${top_builddir}/src/drivers/include
+
+if PNETCDF_DEBUG
+   AM_CPPFLAGS += -DPNETCDF_DEBUG
+endif
+
+if PNETCDF_PROFILING
+   AM_CPPFLAGS += -DPNETCDF_PROFILING
+endif
+
+noinst_LTLIBRARIES = libnczipio.la
+
+M4FLAGS += -I${top_srcdir}/m4
+if ENABLE_ERANGE_FILL
+M4FLAGS += -DERANGE_FILL
+endif
+
+M4_SRCS = nczipioi_profile.m4 \
+          nczipioi_convert.m4
+
+M4H_SRCS = nczipioi_profile.m4h
+
+H_SRCS = nczipio_driver.h 
+
+C_SRCS = nczipio_attr.c \
+         nczipio_dim.c \
+         nczipio_driver.c \
+         nczipio_file.c \
+         nczipio_var.c \
+         nczipio_internal.c \
+         nczipioi_util.c \
+         nczipioi_put_var.c \
+         nczipioi_get_var.c \
+         nczipioi_put_varn.c \
+         nczipioi_get_varn.c \
+         nczipioi_iput_var.c \
+         nczipioi_iget_var.c \
+         nczipioi_iput.c \
+         nczipioi_iget.c \
+         nczipioi_nonblocking.c \
+         nczipioi_cache.c \
+         nczipioi_chunk.c \
+         nczipioi_chunk_size.c \
+         nczipioi_chunk_owner.c \
+         nczipioi_var_init.c \
+         nczipioi_var_resize.c \
+         nczipioi_var_wr.c \
+         nczipioi_var_rd.c \
+         nczipioi_lists.c \
+         nczipioi_nonblocking.c \
+         nczipioi_wait.c \
+         nczip_dummy.c 
+
+if ENABLE_ZLIB
+   C_SRCS += nczip_zlib.c
+endif
+
+if ENABLE_SZ
+   C_SRCS += nczip_sz.c
+endif
+
+$(M4_SRCS:.m4=.c): Makefile
+$(M4H_SRCS:.m4h=.h): Makefile
+
+.m4.c:
+	$(M4) $(AM_M4FLAGS) $(M4FLAGS) $< >$@
+
+.m4h.h:
+	$(M4) $(AM_M4FLAGS) $(M4FLAGS) $< >$@
+
+libnczipio_la_SOURCES = $(C_SRCS) $(H_SRCS)
+nodist_libnczipio_la_SOURCES = $(M4_SRCS:.m4=.c) $(M4H_SRCS:.m4h=.h)
+
+# automake says "... BUILT_SOURCES is honored only by 'make all', 'make check',
+# and 'make install'. This means you cannot build a specific target (e.g.,
+# 'make target') in a clean tree if it depends on a built source."
+BUILT_SOURCES = $(M4_SRCS:.m4=.c) $(M4H_SRCS:.m4h=.h)
+
+CLEANFILES = $(M4_SRCS:.m4=.c) $(M4H_SRCS:.m4h=.h) core core.* *.gcda *.gcno *.gcov gmon.out
+
+EXTRA_DIST = $(M4_HFILES) $(M4_SRCS) $(M4H_SRCS)
+
+tests-local: all
+