Merge pull request #1159 from Unidata/v1.6.0rel

prepare for v1.6.0 release

Author: Jeff Whitaker

.github/workflows/build.yml

@@ -6,7 +6,7 @@ jobs:
     runs-on: ubuntu-latest
     env:
       PNETCDF_VERSION: 1.12.1
-      NETCDF_VERSION: 4.8.0
+      NETCDF_VERSION: 4.9.0
       NETCDF_DIR: ${{ github.workspace }}/..
       NETCDF_EXTRA_CONFIG: --enable-pnetcdf
       CC: mpicc.mpich
@@ -26,7 +26,7 @@ jobs:
     - name: Install Ubuntu Dependencies
       run: |
         sudo apt-get update
-        sudo apt-get install mpich libmpich-dev libhdf5-mpich-dev libcurl4-openssl-dev 
+        sudo apt-get install mpich libmpich-dev libhdf5-mpich-dev libcurl4-openssl-dev bzip2 libsnappy-dev libblosc-dev libzstd-dev
         echo "Download and build PnetCDF version ${PNETCDF_VERSION}"
         wget https://parallel-netcdf.github.io/Release/pnetcdf-${PNETCDF_VERSION}.tar.gz
         tar -xzf pnetcdf-${PNETCDF_VERSION}.tar.gz
@@ -36,7 +36,7 @@ jobs:
         make install
         popd
         echo "Download and build netCDF version ${NETCDF_VERSION}"
-        wget ftp://ftp.unidata.ucar.edu/pub/netcdf/netcdf-c-${NETCDF_VERSION}.tar.gz
+        wget https://downloads.unidata.ucar.edu/netcdf-c/4.9.0/netcdf-c-${NETCDF_VERSION}.tar.gz
         tar -xzf netcdf-c-${NETCDF_VERSION}.tar.gz
         pushd netcdf-c-${NETCDF_VERSION}
         export CPPFLAGS="-I/usr/include/hdf5/mpich -I${NETCDF_DIR}/include"
@@ -61,6 +61,7 @@ jobs:
     - name: Install netcdf4-python
       run: |
         export PATH=${NETCDF_DIR}/bin:${PATH} 
+        export NETCDF_PLUGIN_DIR=${{ github.workspace }}/netcdf-c-${NETCDF_VERSION}/plugins/plugindir
         python setup.py install
     - name: Test
       run: |

.github/workflows/build_master.yml

@@ -23,7 +23,7 @@ jobs:
     - name: Install Ubuntu Dependencies
       run: |
         sudo apt-get update
-        sudo apt-get install mpich libmpich-dev libhdf5-mpich-dev libcurl4-openssl-dev 
+        sudo apt-get install mpich libmpich-dev libhdf5-mpich-dev libcurl4-openssl-dev bzip2 libsnappy-dev libblosc-dev libzstd-dev
         echo "Download and build netCDF github master"
         git clone https://github.com/Unidata/netcdf-c
         pushd netcdf-c
@@ -50,17 +50,18 @@ jobs:
     - name: Install netcdf4-python
       run: |
         export PATH=${NETCDF_DIR}/bin:${PATH} 
+        export NETCDF_PLUGIN_DIR=${{ github.workspace }}/netcdf-c/plugins/plugindir
         python setup.py install
     - name: Test
       run: |
         export PATH=${NETCDF_DIR}/bin:${PATH} 
+        #export HDF5_PLUGIN_PATH=${NETCDF_DIR}/plugins/plugindir
         python checkversion.py
         # serial
         cd test
         python run_all.py
         # parallel
         cd ../examples
-        python bench_compress4.py
         mpirun.mpich -np 4 python mpi_example.py
         if [ $? -ne 0 ] ; then
           echo "hdf5 mpi test failed!"

.github/workflows/miniconda.yml

@@ -78,7 +78,8 @@ jobs:
         export PATH="${CONDA_PREFIX}/bin:${CONDA_PREFIX}/Library/bin:$PATH" 
         which mpirun
         mpirun --version
-        mpirun -np 4 --oversubscribe python mpi_example.py
+        #mpirun -np 4 --oversubscribe python mpi_example.py # for openmpi
+        mpirun -np 4 python mpi_example.py
         if [ $? -ne 0 ] ; then
           echo "hdf5 mpi test failed!"
           exit 1

Changelog

@@ -1,6 +1,6 @@
- version 1.6.0 (not yet released)
-=================================
- * add support for new quantization functionality in netcdf-c 4.8.2 via "signficant_digits"
+ version 1.6.0 (tag v1.6.0rel)
+==============================
+ * add support for new quantization functionality in netcdf-c 4.9.0 via "signficant_digits"
    and "quantize_mode" kwargs in Dataset.createVariable. Default quantization_mode is "BitGroom",
    but alternate methods "BitRound" and GranularBitRound" also supported.
  * opening a Dataset in append mode (mode = 'a' or 'r+') creates a Dataset
@@ -11,10 +11,18 @@
    names in "dimensions" tuple kwarg (issue #1145).
  * remove all vestiges of python 2 in _netCDF4.pyx and set cython language_level
    directive to 3 in setup.py.
- * add 'compression' kwarg to createVariable.  Only 'None' and 'zlib' currently
-   allowed (compression='zlib' is equivalent to zlib=True), but allows
-   for new compression algorithms to be added when they become available
-   in netcdf-c. The 'zlib' kwarg is now deprecated.
+ * add 'compression' kwarg to createVariable to enable new compression
+   functionality in netcdf-c 4.9.0.  'None','zlib','szip','zstd','bzip2'
+   'blosc_lz','blosc_lz4','blosc_lz4hc','blosc_zlib' and 'blosc_zstd'
+   are currently supported. 'blosc_shuffle',
+   'szip_mask' and 'szip_pixels_per_block' kwargs also added.
+   compression='zlib' is equivalent to (the now deprecated) zlib=True.
+   If the environment variable NETCDF_PLUGIN_DIR is set to point to the
+   directory with the compression plugin lib__nc* files, then the compression plugins will
+   be installed within the package and be automatically available (the binary
+   wheels have this).  Otherwise, the environment variable HDF5_PLUGIN_PATH
+   needs to be set at runtime  to point to plugins in order to use the new compression
+   options.
  * MFDataset did not aggregate 'name' variable attribute (issue #1153).
  * issue warning instead of raising an exception if missing_value or
    _FillValue can't be cast to the variable type when creating a 

MANIFEST.in

@@ -13,6 +13,7 @@ include src/netCDF4/__init__.py
 include src/netCDF4/_netCDF4.pyx
 exclude src/netCDF4/_netCDF4.c
 include src/netCDF4/utils.py
+include src/netCDF4/plugins/empty.txt
 include include/netCDF4.pxi
 include include/mpi-compat.h
 include include/membuf.pyx

README.md

@@ -10,6 +10,13 @@
 ## News
 For details on the latest updates, see the [Changelog](https://github.com/Unidata/netcdf4-python/blob/master/Changelog).
 
+??/??/2022:  Version [1.6.0](https://pypi.python.org/pypi/netCDF4/1.6.0) released.  Support
+for quantization (bit-grooming and bit-rounding) functionality in netcdf-c 4.9.0 which can
+dramatically improve compression.  Dataset.createVariable now accepts dimension instances (instead
+of just dimension names). 'compression' kwarg added to Dataset.createVariable to support szip as
+well as new compression algorithms available in netcdf-c 4.9.0 through compression plugins (such
+as zstd, bzip2 and blosc). Working arm64 wheels for Apple M1 Silicon now available on pypi.
+
 10/31/2021:  Version [1.5.8](https://pypi.python.org/pypi/netCDF4/1.5.8) released. Fix Enum bug, add binary wheels for aarch64 and python 3.10.
 
 6/22/2021:  Version [1.5.7](https://pypi.python.org/pypi/netCDF4/1.5.7) released.