Parallel-NetCDF · wkliao · Mar 23, 2025 · Mar 18, 2025 · Mar 18, 2025 · Mar 18, 2025
diff --git a/.github/workflows/cache_async_vol.yml b/.github/workflows/cache_async_vol.yml
@@ -19,10 +19,10 @@ on:
       - 'tests/*'
 
 env:
-   MPICH_VERSION: 4.1.1
+   MPICH_VERSION: 4.3.0
    ARGOBOTS_VERSION: 1.1
    ASYNC_VOL_VERSION: 1.8.1
-   HDF5_VERSION: 1.14.2
+   HDF5_VERSION: 1.14.6
    LOG_VOL_VERSION: 1.4.0
 
 jobs:
@@ -92,8 +92,11 @@ jobs:
             cd ${GITHUB_WORKSPACE}
             rm -rf ${HDF5_ROOT} ; mkdir ${HDF5_ROOT} ; cd ${HDF5_ROOT}
             VER_MAJOR=${HDF5_VERSION%.*}
+            VER_MAJOR=${VER_MAJOR/./_} # replace . with _
             VER_NOPATCH=${HDF5_VERSION%-*} # remove patch version
-            curl -LO https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-${VER_MAJOR}/hdf5-${VER_NOPATCH}/src/hdf5-${HDF5_VERSION}.tar.gz
+            VER_NOPATCH=${VER_NOPATCH//./_}
+            # curl -LO https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-${VER_MAJOR}/hdf5-${VER_NOPATCH}/src/hdf5-${HDF5_VERSION}.tar.gz
+            curl -LO https://support.hdfgroup.org/releases/hdf5/v${VER_MAJOR}/v${VER_NOPATCH}/downloads/hdf5-${HDF5_VERSION}.tar.gz
             tar -zxf hdf5-${HDF5_VERSION}.tar.gz
             cd hdf5-${HDF5_VERSION}
             ./configure --prefix=${HDF5_ROOT} \

diff --git a/.github/workflows/hdf5_mpich.yml b/.github/workflows/hdf5_mpich.yml
@@ -0,0 +1,145 @@
+name: HDF5 and MPICH
+
+on:
+  push:
+    branches: [ master ]
+    paths-ignore:
+      - '**/*.md'
+      - '**/*.txt'
+      - '**/*.jpg'
+      - '**/*.png'
+      - 'tests/*'
+  pull_request:
+    branches: [ master ]
+    paths-ignore:
+      - '**/*.md'
+      - '**/*.txt'
+      - '**/*.jpg'
+      - '**/*.png'
+      - 'tests/*'
+
+env:
+   MPICH_VERSION: 4.3.0
+   HDF5_VERSION: 1.14.6
+
+jobs:
+    build:
+      runs-on: ubuntu-latest
+      timeout-minutes: 60
+      steps:
+        - uses: actions/checkout@v4
+        - name: Set up dependencies
+          run: |
+            set -x
+            sudo apt-get update
+            sudo apt-get -y install automake autoconf libtool libtool-bin m4 cmake
+            # The MPICH installed on github actions is too slow
+            # sudo apt-get install mpich
+            # mpicc -v
+            # zlib
+            sudo apt-get -y install zlib1g-dev
+        - name: Add global env variables into GITHUB_ENV
+          run: |
+            set -x
+            echo "MPICH_DIR=${GITHUB_WORKSPACE}/MPICH" >> $GITHUB_ENV
+            echo "HDF5_ROOT=${GITHUB_WORKSPACE}/HDF5" >> $GITHUB_ENV
+        - name: Build MPICH ${{ env.MPICH_VERSION }}
+          if: ${{ success() }}
+          run: |
+            set -x
+            cd ${GITHUB_WORKSPACE}
+            rm -rf ${MPICH_DIR} ; mkdir ${MPICH_DIR} ; cd ${MPICH_DIR}
+            curl -LO https://www.mpich.org/static/downloads/${MPICH_VERSION}/mpich-${MPICH_VERSION}.tar.gz
+            gzip -dc mpich-${MPICH_VERSION}.tar.gz | tar -xf -
+            cd mpich-${MPICH_VERSION}
+            ./configure --prefix=${MPICH_DIR} \
+                        --silent \
+                        --enable-romio \
+                        --with-file-system=ufs \
+                        --with-device=ch3:sock \
+                        --disable-fortran \
+                        CC=gcc
+            make -s LIBTOOLFLAGS=--silent V=1 -j 8 install > qout 2>&1
+            make -s -j 8 distclean >> qout 2>&1
+        - name: Dump MPICH log file
+          if: ${{ failure() }}
+          run: |
+            set -x
+            cat ${MPICH_DIR}/mpich-${MPICH_VERSION}/qout
+            cat ${MPICH_DIR}/mpich-${MPICH_VERSION}/config.log
+        - name: Install HDF5 ${{ env.HDF5_VERSION }}
+          if: ${{ success() }}
+          run: |
+            set -x
+            cd ${GITHUB_WORKSPACE}
+            rm -rf ${HDF5_ROOT} ; mkdir ${HDF5_ROOT} ; cd ${HDF5_ROOT}
+            VER_MAJOR=${HDF5_VERSION%.*}
+            VER_MAJOR=${VER_MAJOR/./_} # replace . with _
+            VER_NOPATCH=${HDF5_VERSION%-*} # remove patch version
+            VER_NOPATCH=${VER_NOPATCH//./_}
+            # curl -LO https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-${VER_MAJOR}/hdf5-${VER_NOPATCH}/src/hdf5-${HDF5_VERSION}.tar.gz
+            curl -LO https://support.hdfgroup.org/releases/hdf5/v${VER_MAJOR}/v${VER_NOPATCH}/downloads/hdf5-${HDF5_VERSION}.tar.gz
+            tar -zxf hdf5-${HDF5_VERSION}.tar.gz
+            mkdir build
+            cd build
+            CC=${MPICH_DIR}/bin/mpicc cmake ../hdf5-${HDF5_VERSION} \
+                                            -D"CMAKE_INSTALL_PREFIX=${HDF5_ROOT}" \
+                                            -D"HDF5_ENABLE_PARALLEL=ON"
+            make -j 8 install > qout 2>&1
+        - name: Dump config.log file if build HDF5 failed
+          if: ${{ failure() }}
+          run: |
+            set -x
+            cd ${GITHUB_WORKSPACE}
+            cat ${HDF5_ROOT}/hdf5-${HDF5_VERSION}/qout
+            cat ${HDF5_ROOT}/hdf5-${HDF5_VERSION}/config.log
+        - name: Build E3SM_IO with HDF5
+          if: ${{ success() }}
+          run: |
+            set -x
+            cd ${GITHUB_WORKSPACE}
+            rm -rf ./test_output
+            autoreconf -i
+            ./configure --with-mpi=${MPICH_DIR} \
+                        --with-hdf5=${HDF5_ROOT} \
+                        CFLAGS=-fno-var-tracking-assignments \
+                        CXXFLAGS=-fno-var-tracking-assignments
+            make -j 8
+        - name: Print config.log if error
+          if: ${{ failure() }}
+          run: |
+            set -x
+            cat ${GITHUB_WORKSPACE}/config.log
+        - name: Test - make check
+          if: ${{ success() }}
+          run: |
+            set -x
+            cd ${GITHUB_WORKSPACE}
+            make -s check
+        - name: Print log files
+          if: ${{ always() }}
+          run: |
+            set -x
+            cd ${GITHUB_WORKSPACE}
+            cat test.sh.log
+            cat utils/*.log
+        - name: Test - make ptest
+          if: ${{ success() }}
+          run: |
+            set -x
+            cd ${GITHUB_WORKSPACE}
+            make -s ptest
+        - name: Print log files
+          if: ${{ always() }}
+          run: |
+            set -x
+            cd ${GITHUB_WORKSPACE}
+            cat test.sh.log
+            cat utils/*.log
+        - name: make distclean
+          if: ${{ always() }}
+          run: |
+            set -x
+            cd ${GITHUB_WORKSPACE}
+            make -s distclean
+
diff --git a/.github/workflows/logvol_master.yml b/.github/workflows/logvol_master.yml
@@ -23,7 +23,7 @@ on:
       - 'datasets/*'
 
 env:
-   HDF5_VERSION: 1.14.2
+   HDF5_VERSION: 1.14.6
 
 jobs:
     build:
@@ -50,8 +50,11 @@ jobs:
             cd ${GITHUB_WORKSPACE}
             rm -rf HDF5 ; mkdir HDF5 ; cd HDF5
             VER_MAJOR=${HDF5_VERSION%.*}
+            VER_MAJOR=${VER_MAJOR/./_} # replace . with _
             VER_NOPATCH=${HDF5_VERSION%-*} # remove patch version
-            curl -LO https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-${VER_MAJOR}/hdf5-${VER_NOPATCH}/src/hdf5-${HDF5_VERSION}.tar.gz
+            VER_NOPATCH=${VER_NOPATCH//./_}
+            # curl -LO https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-${VER_MAJOR}/hdf5-${VER_NOPATCH}/src/hdf5-${HDF5_VERSION}.tar.gz
+            curl -LO https://support.hdfgroup.org/releases/hdf5/v${VER_MAJOR}/v${VER_NOPATCH}/downloads/hdf5-${HDF5_VERSION}.tar.gz
             tar -zxf hdf5-${HDF5_VERSION}.tar.gz
             cd hdf5-${HDF5_VERSION}
             ./configure --prefix=${GITHUB_WORKSPACE}/HDF5 \

diff --git a/.github/workflows/mpich_static.yml b/.github/workflows/mpich_static.yml
@@ -23,9 +23,9 @@ on:
       - 'datasets/*'
 
 env:
-   MPICH_VERSION: 4.2.3
+   MPICH_VERSION: 4.3.0
    PNETCDF_VERSION: 1.14.0
-   HDF5_VERSION: 1.14.2
+   HDF5_VERSION: 1.14.6
    NETCDF4_VERSION: 4.9.2
    ADIOS_VERSION: 2.8.3
    LOG_VOL_VERSION: 1.4.0
@@ -104,8 +104,11 @@ jobs:
             echo "Install HDF5 on ${GITHUB_WORKSPACE}/HDF5"
             rm -rf HDF5 ; mkdir HDF5 ; cd HDF5
             VER_MAJOR=${HDF5_VERSION%.*}
+            VER_MAJOR=${VER_MAJOR/./_} # replace . with _
             VER_NOPATCH=${HDF5_VERSION%-*} # remove patch version
-            curl -LO https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-${VER_MAJOR}/hdf5-${VER_NOPATCH}/src/hdf5-${HDF5_VERSION}.tar.gz
+            VER_NOPATCH=${VER_NOPATCH//./_}
+            # curl -LO https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-${VER_MAJOR}/hdf5-${VER_NOPATCH}/src/hdf5-${HDF5_VERSION}.tar.gz
+            curl -LO https://support.hdfgroup.org/releases/hdf5/v${VER_MAJOR}/v${VER_NOPATCH}/downloads/hdf5-${HDF5_VERSION}.tar.gz
             tar -zxf hdf5-${HDF5_VERSION}.tar.gz
             cd hdf5-${HDF5_VERSION}
             ./configure --prefix=${GITHUB_WORKSPACE}/HDF5 \

diff --git a/.github/workflows/pnetcdf_master.yml b/.github/workflows/pnetcdf_master.yml
@@ -23,7 +23,11 @@ on:
       - 'datasets/*'
 
 env:
-   MPICH_VERSION: 4.2.3
+   MPICH_VERSION: 4.3.0
+   AUTOCONF_VERSION: 2.71
+   AUTOMAKE_VERSION: 1.17
+   M4_VERSION: 1.4.19
+   LIBTOOL_VERSION: 2.5.4
    PNETCDF_VERSION: repo
 
 jobs:
@@ -36,11 +40,46 @@ jobs:
           run: |
             set -x
             sudo apt-get update
-            sudo apt-get install automake autoconf libtool libtool-bin m4
             # mpich
             # sudo apt-get install mpich
             # zlib
             sudo apt-get install zlib1g-dev
+        - name: Build GNU autotools
+          run: |
+            export PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/bin:${PATH}"
+            export LD_LIBRARY_PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/lib:${LD_LIBRARY_PATH}"
+            cd ${GITHUB_WORKSPACE}
+            wget -q https://ftp.gnu.org/gnu/m4/m4-${M4_VERSION}.tar.gz
+            gzip -dc m4-${M4_VERSION}.tar.gz | tar -xf -
+            cd m4-${M4_VERSION}
+            ./configure --prefix=${GITHUB_WORKSPACE}/AUTOTOOLS \
+                        --silent
+            make -s -j 8 install > qout 2>&1
+            make -s -j 8 distclean >> qout 2>&1
+            cd ${GITHUB_WORKSPACE}
+            wget -q https://ftp.gnu.org/gnu/autoconf/autoconf-${AUTOCONF_VERSION}.tar.gz
+            gzip -dc autoconf-${AUTOCONF_VERSION}.tar.gz | tar -xf -
+            cd autoconf-${AUTOCONF_VERSION}
+            ./configure --prefix=${GITHUB_WORKSPACE}/AUTOTOOLS \
+                        --silent
+            make -s -j 8 install > qout 2>&1
+            make -s -j 8 distclean >> qout 2>&1
+            cd ${GITHUB_WORKSPACE}
+            wget -q https://ftp.gnu.org/gnu/automake/automake-${AUTOMAKE_VERSION}.tar.gz
+            gzip -dc automake-${AUTOMAKE_VERSION}.tar.gz | tar -xf -
+            cd automake-${AUTOMAKE_VERSION}
+            ./configure --prefix=${GITHUB_WORKSPACE}/AUTOTOOLS \
+                        --silent
+            make -s -j 8 install > qout 2>&1
+            make -s -j 8 distclean >> qout 2>&1
+            cd ${GITHUB_WORKSPACE}
+            wget -q https://ftp.gnu.org/gnu/libtool/libtool-${LIBTOOL_VERSION}.tar.gz
+            gzip -dc libtool-${LIBTOOL_VERSION}.tar.gz | tar -xf -
+            cd libtool-${LIBTOOL_VERSION}
+            ./configure --prefix=${GITHUB_WORKSPACE}/AUTOTOOLS \
+                        --silent
+            make -s -j 8 install > qout 2>&1
+            make -s -j 8 distclean >> qout 2>&1
         - name: Build MPICH
           run: |
             set -x
@@ -73,6 +112,12 @@ jobs:
           run: |
             set -x
             cd ${GITHUB_WORKSPACE}
+            export PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/bin:${PATH}"
+            export LD_LIBRARY_PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/lib:${LD_LIBRARY_PATH}"
+            m4 --version
+            autoconf --version
+            automake --version
+            libtool --version
             echo "Install PnetCDF on ${GITHUB_WORKSPACE}/PnetCDF"
             rm -rf PnetCDF ; mkdir PnetCDF ; cd PnetCDF
             # curl -LO https://parallel-netcdf.github.io/Release/pnetcdf-${PNETCDF_VERSION}.tar.gz

diff --git a/.github/workflows/ubuntu_ompi.yml b/.github/workflows/ubuntu_ompi.yml
@@ -24,7 +24,7 @@ on:
 
 env:
    PNETCDF_VERSION: 1.14.0
-   HDF5_VERSION: 1.14.2
+   HDF5_VERSION: 1.14.6
    NETCDF4_VERSION: 4.9.2
    ADIOS_VERSION: 2.8.3
    LOG_VOL_VERSION: 1.4.0
@@ -75,8 +75,11 @@ jobs:
             echo "Install HDF5 on ${GITHUB_WORKSPACE}/HDF5"
             rm -rf HDF5 ; mkdir HDF5 ; cd HDF5
             VER_MAJOR=${HDF5_VERSION%.*}
+            VER_MAJOR=${VER_MAJOR/./_} # replace . with _
             VER_NOPATCH=${HDF5_VERSION%-*} # remove patch version
-            curl -LO https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-${VER_MAJOR}/hdf5-${VER_NOPATCH}/src/hdf5-${HDF5_VERSION}.tar.gz
+            VER_NOPATCH=${VER_NOPATCH//./_}
+            # curl -LO https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-${VER_MAJOR}/hdf5-${VER_NOPATCH}/src/hdf5-${HDF5_VERSION}.tar.gz
+            curl -LO https://support.hdfgroup.org/releases/hdf5/v${VER_MAJOR}/v${VER_NOPATCH}/downloads/hdf5-${HDF5_VERSION}.tar.gz
             tar -zxf hdf5-${HDF5_VERSION}.tar.gz
             cd hdf5-${HDF5_VERSION}
             ./configure --prefix=${GITHUB_WORKSPACE}/HDF5 \

diff --git a/configure.ac b/configure.ac
@@ -807,14 +807,51 @@ if test "x$have_hdf5" = xyes ; then
       extra_libs="`grep 'Extra libraries' $hdf5_lib/libhdf5.settings |cut -f2 -d:`"
       AC_MSG_RESULT([$extra_libs])
    fi
-   if test "x$LIBS" = x ; then
-      LIBS="-lhdf5 $extra_libs"
-   else
+
+   # Building HDF5 1.14.6 using cmake produces an incorrect results in the field of
+   # "Extra libraries". Test it first before including it to LIBS.
+   if test "x$extra_libs" != x ; then
+      saved_LIBS=$LIBS
       LIBS="-lhdf5 $extra_libs $LIBS"
+      AC_MSG_CHECKING([extra libraries shown in libhdf5.settings if valid])
+      AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include <hdf5.h>]],
+                     [[H5Fopen("dummy", H5F_ACC_RDWR, H5P_DEFAULT);]])],
+                     [valid_extra_libs=yes], [valid_extra_libs=no])
+      AC_MSG_RESULT([$valid_extra_libs])
+      if test "x$valid_extra_libs" = xno && test -d "$hdf5_lib/pkgconfig" ; then
+         # extra_libs is not valid, check pkg-config
+         saved_PKG_CONFIG_PATH=$PKG_CONFIG_PATH
+         export PKG_CONFIG_PATH="$hdf5_lib/pkgconfig:$PKG_CONFIG_PATH"
+         extra_ldflags=`pkg-config --libs-only-L hdf5 2> conftest.err`
+         rm -f conftest.err
+         extra_libs=`pkg-config --libs-only-l hdf5 2> conftest.err`
+         rm -f conftest.err
+         export PKG_CONFIG_PATH=$saved_PKG_CONFIG_PATH
+         if test "x$extra_ldflags" != x || test "x$extra_libs" != x ; then
+            saved_LDFLAGS=$LDFLAGS
+            LDFLAGS="$extra_libs $LDFLAGS"
+            LIBS="-lhdf5 $extra_libs $saved_LIBS"
+            AC_MSG_CHECKING([extra libraries shown in libhdf5.settings if valid])
+            AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include <hdf5.h>]],
+                           [[H5Fopen("dummy", H5F_ACC_RDWR, H5P_DEFAULT);]])],
+                           [valid_extra_libs=yes], [valid_extra_libs=no])
+            AC_MSG_RESULT([$valid_extra_libs])
+            if test "x$valid_extra_libs" = xno ; then
+               extra_ldflags=
+               extra_libs=
+            fi
+            LDFLAGS=$saved_LDFLAGS
+         fi
+      fi
+      LIBS=$saved_LIBS
    fi
-   if test "x$LDFLAGS" = x ; then
-      LDFLAGS=$extra_ldflags
-   else
+
+   LIBS="$extra_libs $LIBS"
+   if ! echo $LIBS | grep -q "\-lhdf5" ; then
+      LIBS="-lhdf5 $LIBS"
+   fi
+   LDFLAGS="$extra_ldflags $LDFLAGS"
+   if ! echo $LDFLAGS | grep -q "\-L$hdf5_lib" ; then
       LDFLAGS="$extra_ldflags $LDFLAGS"
    fi
    AC_DEFINE([ENABLE_HDF5], [1], [Enable HDF5 I/O method])