diff --git a/.github/workflows/cache_async_vol.yml b/.github/workflows/cache_async_vol.yml index e66f9753..c5a437f7 100644 --- a/.github/workflows/cache_async_vol.yml +++ b/.github/workflows/cache_async_vol.yml @@ -19,10 +19,10 @@ on: - 'tests/*' env: - MPICH_VERSION: 4.1.1 + MPICH_VERSION: 4.3.0 ARGOBOTS_VERSION: 1.1 ASYNC_VOL_VERSION: 1.8.1 - HDF5_VERSION: 1.14.2 + HDF5_VERSION: 1.14.6 LOG_VOL_VERSION: 1.4.0 jobs: @@ -92,8 +92,11 @@ jobs: cd ${GITHUB_WORKSPACE} rm -rf ${HDF5_ROOT} ; mkdir ${HDF5_ROOT} ; cd ${HDF5_ROOT} VER_MAJOR=${HDF5_VERSION%.*} + VER_MAJOR=${VER_MAJOR/./_} # replace . with _ VER_NOPATCH=${HDF5_VERSION%-*} # remove patch version - curl -LO https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-${VER_MAJOR}/hdf5-${VER_NOPATCH}/src/hdf5-${HDF5_VERSION}.tar.gz + VER_NOPATCH=${VER_NOPATCH//./_} + # curl -LO https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-${VER_MAJOR}/hdf5-${VER_NOPATCH}/src/hdf5-${HDF5_VERSION}.tar.gz + curl -LO https://support.hdfgroup.org/releases/hdf5/v${VER_MAJOR}/v${VER_NOPATCH}/downloads/hdf5-${HDF5_VERSION}.tar.gz tar -zxf hdf5-${HDF5_VERSION}.tar.gz cd hdf5-${HDF5_VERSION} ./configure --prefix=${HDF5_ROOT} \ diff --git a/.github/workflows/hdf5_mpich.yml b/.github/workflows/hdf5_mpich.yml new file mode 100644 index 00000000..76fc03e7 --- /dev/null +++ b/.github/workflows/hdf5_mpich.yml @@ -0,0 +1,145 @@ +name: HDF5 and MPICH + +on: + push: + branches: [ master ] + paths-ignore: + - '**/*.md' + - '**/*.txt' + - '**/*.jpg' + - '**/*.png' + - 'tests/*' + pull_request: + branches: [ master ] + paths-ignore: + - '**/*.md' + - '**/*.txt' + - '**/*.jpg' + - '**/*.png' + - 'tests/*' + +env: + MPICH_VERSION: 4.3.0 + HDF5_VERSION: 1.14.6 + +jobs: + build: + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + - name: Set up dependencies + run: | + set -x + sudo apt-get update + sudo apt-get -y install automake autoconf libtool libtool-bin m4 cmake + # The MPICH installed on github actions is too slow + # sudo apt-get install mpich + # mpicc -v + # zlib + sudo apt-get -y install zlib1g-dev + - name: Add global env variables into GITHUB_ENV + run: | + set -x + echo "MPICH_DIR=${GITHUB_WORKSPACE}/MPICH" >> $GITHUB_ENV + echo "HDF5_ROOT=${GITHUB_WORKSPACE}/HDF5" >> $GITHUB_ENV + - name: Build MPICH ${{ env.MPICH_VERSION }} + if: ${{ success() }} + run: | + set -x + cd ${GITHUB_WORKSPACE} + rm -rf ${MPICH_DIR} ; mkdir ${MPICH_DIR} ; cd ${MPICH_DIR} + curl -LO https://www.mpich.org/static/downloads/${MPICH_VERSION}/mpich-${MPICH_VERSION}.tar.gz + gzip -dc mpich-${MPICH_VERSION}.tar.gz | tar -xf - + cd mpich-${MPICH_VERSION} + ./configure --prefix=${MPICH_DIR} \ + --silent \ + --enable-romio \ + --with-file-system=ufs \ + --with-device=ch3:sock \ + --disable-fortran \ + CC=gcc + make -s LIBTOOLFLAGS=--silent V=1 -j 8 install > qout 2>&1 + make -s -j 8 distclean >> qout 2>&1 + - name: Dump MPICH log file + if: ${{ failure() }} + run: | + set -x + cat ${MPICH_DIR}/mpich-${MPICH_VERSION}/qout + cat ${MPICH_DIR}/mpich-${MPICH_VERSION}/config.log + - name: Install HDF5 ${{ env.HDF5_VERSION }} + if: ${{ success() }} + run: | + set -x + cd ${GITHUB_WORKSPACE} + rm -rf ${HDF5_ROOT} ; mkdir ${HDF5_ROOT} ; cd ${HDF5_ROOT} + VER_MAJOR=${HDF5_VERSION%.*} + VER_MAJOR=${VER_MAJOR/./_} # replace . with _ + VER_NOPATCH=${HDF5_VERSION%-*} # remove patch version + VER_NOPATCH=${VER_NOPATCH//./_} + # curl -LO https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-${VER_MAJOR}/hdf5-${VER_NOPATCH}/src/hdf5-${HDF5_VERSION}.tar.gz + curl -LO https://support.hdfgroup.org/releases/hdf5/v${VER_MAJOR}/v${VER_NOPATCH}/downloads/hdf5-${HDF5_VERSION}.tar.gz + tar -zxf hdf5-${HDF5_VERSION}.tar.gz + mkdir build + cd build + CC=${MPICH_DIR}/bin/mpicc cmake ../hdf5-${HDF5_VERSION} \ + -D"CMAKE_INSTALL_PREFIX=${HDF5_ROOT}" \ + -D"HDF5_ENABLE_PARALLEL=ON" + make -j 8 install > qout 2>&1 + - name: Dump config.log file if build HDF5 failed + if: ${{ failure() }} + run: | + set -x + cd ${GITHUB_WORKSPACE} + cat ${HDF5_ROOT}/hdf5-${HDF5_VERSION}/qout + cat ${HDF5_ROOT}/hdf5-${HDF5_VERSION}/config.log + - name: Build E3SM_IO with HDF5 + if: ${{ success() }} + run: | + set -x + cd ${GITHUB_WORKSPACE} + rm -rf ./test_output + autoreconf -i + ./configure --with-mpi=${MPICH_DIR} \ + --with-hdf5=${HDF5_ROOT} \ + CFLAGS=-fno-var-tracking-assignments \ + CXXFLAGS=-fno-var-tracking-assignments + make -j 8 + - name: Print config.log if error + if: ${{ failure() }} + run: | + set -x + cat ${GITHUB_WORKSPACE}/config.log + - name: Test - make check + if: ${{ success() }} + run: | + set -x + cd ${GITHUB_WORKSPACE} + make -s check + - name: Print log files + if: ${{ always() }} + run: | + set -x + cd ${GITHUB_WORKSPACE} + cat test.sh.log + cat utils/*.log + - name: Test - make ptest + if: ${{ success() }} + run: | + set -x + cd ${GITHUB_WORKSPACE} + make -s ptest + - name: Print log files + if: ${{ always() }} + run: | + set -x + cd ${GITHUB_WORKSPACE} + cat test.sh.log + cat utils/*.log + - name: make distclean + if: ${{ always() }} + run: | + set -x + cd ${GITHUB_WORKSPACE} + make -s distclean + diff --git a/.github/workflows/logvol_master.yml b/.github/workflows/logvol_master.yml index dbd40eca..d7276706 100644 --- a/.github/workflows/logvol_master.yml +++ b/.github/workflows/logvol_master.yml @@ -23,7 +23,7 @@ on: - 'datasets/*' env: - HDF5_VERSION: 1.14.2 + HDF5_VERSION: 1.14.6 jobs: build: @@ -50,8 +50,11 @@ jobs: cd ${GITHUB_WORKSPACE} rm -rf HDF5 ; mkdir HDF5 ; cd HDF5 VER_MAJOR=${HDF5_VERSION%.*} + VER_MAJOR=${VER_MAJOR/./_} # replace . with _ VER_NOPATCH=${HDF5_VERSION%-*} # remove patch version - curl -LO https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-${VER_MAJOR}/hdf5-${VER_NOPATCH}/src/hdf5-${HDF5_VERSION}.tar.gz + VER_NOPATCH=${VER_NOPATCH//./_} + # curl -LO https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-${VER_MAJOR}/hdf5-${VER_NOPATCH}/src/hdf5-${HDF5_VERSION}.tar.gz + curl -LO https://support.hdfgroup.org/releases/hdf5/v${VER_MAJOR}/v${VER_NOPATCH}/downloads/hdf5-${HDF5_VERSION}.tar.gz tar -zxf hdf5-${HDF5_VERSION}.tar.gz cd hdf5-${HDF5_VERSION} ./configure --prefix=${GITHUB_WORKSPACE}/HDF5 \ diff --git a/.github/workflows/mpich_static.yml b/.github/workflows/mpich_static.yml index e901ebe3..e92ef652 100644 --- a/.github/workflows/mpich_static.yml +++ b/.github/workflows/mpich_static.yml @@ -23,9 +23,9 @@ on: - 'datasets/*' env: - MPICH_VERSION: 4.2.3 + MPICH_VERSION: 4.3.0 PNETCDF_VERSION: 1.14.0 - HDF5_VERSION: 1.14.2 + HDF5_VERSION: 1.14.6 NETCDF4_VERSION: 4.9.2 ADIOS_VERSION: 2.8.3 LOG_VOL_VERSION: 1.4.0 @@ -104,8 +104,11 @@ jobs: echo "Install HDF5 on ${GITHUB_WORKSPACE}/HDF5" rm -rf HDF5 ; mkdir HDF5 ; cd HDF5 VER_MAJOR=${HDF5_VERSION%.*} + VER_MAJOR=${VER_MAJOR/./_} # replace . with _ VER_NOPATCH=${HDF5_VERSION%-*} # remove patch version - curl -LO https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-${VER_MAJOR}/hdf5-${VER_NOPATCH}/src/hdf5-${HDF5_VERSION}.tar.gz + VER_NOPATCH=${VER_NOPATCH//./_} + # curl -LO https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-${VER_MAJOR}/hdf5-${VER_NOPATCH}/src/hdf5-${HDF5_VERSION}.tar.gz + curl -LO https://support.hdfgroup.org/releases/hdf5/v${VER_MAJOR}/v${VER_NOPATCH}/downloads/hdf5-${HDF5_VERSION}.tar.gz tar -zxf hdf5-${HDF5_VERSION}.tar.gz cd hdf5-${HDF5_VERSION} ./configure --prefix=${GITHUB_WORKSPACE}/HDF5 \ diff --git a/.github/workflows/pnetcdf_master.yml b/.github/workflows/pnetcdf_master.yml index f5646b1d..351f3013 100644 --- a/.github/workflows/pnetcdf_master.yml +++ b/.github/workflows/pnetcdf_master.yml @@ -23,7 +23,11 @@ on: - 'datasets/*' env: - MPICH_VERSION: 4.2.3 + MPICH_VERSION: 4.3.0 + AUTOCONF_VERSION: 2.71 + AUTOMAKE_VERSION: 1.17 + M4_VERSION: 1.4.19 + LIBTOOL_VERSION: 2.5.4 PNETCDF_VERSION: repo jobs: @@ -36,11 +40,46 @@ jobs: run: | set -x sudo apt-get update - sudo apt-get install automake autoconf libtool libtool-bin m4 # mpich # sudo apt-get install mpich # zlib sudo apt-get install zlib1g-dev + - name: Build GNU autotools + run: | + export PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/bin:${PATH}" + export LD_LIBRARY_PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/lib:${LD_LIBRARY_PATH}" + cd ${GITHUB_WORKSPACE} + wget -q https://ftp.gnu.org/gnu/m4/m4-${M4_VERSION}.tar.gz + gzip -dc m4-${M4_VERSION}.tar.gz | tar -xf - + cd m4-${M4_VERSION} + ./configure --prefix=${GITHUB_WORKSPACE}/AUTOTOOLS \ + --silent + make -s -j 8 install > qout 2>&1 + make -s -j 8 distclean >> qout 2>&1 + cd ${GITHUB_WORKSPACE} + wget -q https://ftp.gnu.org/gnu/autoconf/autoconf-${AUTOCONF_VERSION}.tar.gz + gzip -dc autoconf-${AUTOCONF_VERSION}.tar.gz | tar -xf - + cd autoconf-${AUTOCONF_VERSION} + ./configure --prefix=${GITHUB_WORKSPACE}/AUTOTOOLS \ + --silent + make -s -j 8 install > qout 2>&1 + make -s -j 8 distclean >> qout 2>&1 + cd ${GITHUB_WORKSPACE} + wget -q https://ftp.gnu.org/gnu/automake/automake-${AUTOMAKE_VERSION}.tar.gz + gzip -dc automake-${AUTOMAKE_VERSION}.tar.gz | tar -xf - + cd automake-${AUTOMAKE_VERSION} + ./configure --prefix=${GITHUB_WORKSPACE}/AUTOTOOLS \ + --silent + make -s -j 8 install > qout 2>&1 + make -s -j 8 distclean >> qout 2>&1 + cd ${GITHUB_WORKSPACE} + wget -q https://ftp.gnu.org/gnu/libtool/libtool-${LIBTOOL_VERSION}.tar.gz + gzip -dc libtool-${LIBTOOL_VERSION}.tar.gz | tar -xf - + cd libtool-${LIBTOOL_VERSION} + ./configure --prefix=${GITHUB_WORKSPACE}/AUTOTOOLS \ + --silent + make -s -j 8 install > qout 2>&1 + make -s -j 8 distclean >> qout 2>&1 - name: Build MPICH run: | set -x @@ -73,6 +112,12 @@ jobs: run: | set -x cd ${GITHUB_WORKSPACE} + export PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/bin:${PATH}" + export LD_LIBRARY_PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/lib:${LD_LIBRARY_PATH}" + m4 --version + autoconf --version + automake --version + libtool --version echo "Install PnetCDF on ${GITHUB_WORKSPACE}/PnetCDF" rm -rf PnetCDF ; mkdir PnetCDF ; cd PnetCDF # curl -LO https://parallel-netcdf.github.io/Release/pnetcdf-${PNETCDF_VERSION}.tar.gz diff --git a/.github/workflows/ubuntu_ompi.yml b/.github/workflows/ubuntu_ompi.yml index 473d9e83..555d5ab3 100644 --- a/.github/workflows/ubuntu_ompi.yml +++ b/.github/workflows/ubuntu_ompi.yml @@ -24,7 +24,7 @@ on: env: PNETCDF_VERSION: 1.14.0 - HDF5_VERSION: 1.14.2 + HDF5_VERSION: 1.14.6 NETCDF4_VERSION: 4.9.2 ADIOS_VERSION: 2.8.3 LOG_VOL_VERSION: 1.4.0 @@ -75,8 +75,11 @@ jobs: echo "Install HDF5 on ${GITHUB_WORKSPACE}/HDF5" rm -rf HDF5 ; mkdir HDF5 ; cd HDF5 VER_MAJOR=${HDF5_VERSION%.*} + VER_MAJOR=${VER_MAJOR/./_} # replace . with _ VER_NOPATCH=${HDF5_VERSION%-*} # remove patch version - curl -LO https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-${VER_MAJOR}/hdf5-${VER_NOPATCH}/src/hdf5-${HDF5_VERSION}.tar.gz + VER_NOPATCH=${VER_NOPATCH//./_} + # curl -LO https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-${VER_MAJOR}/hdf5-${VER_NOPATCH}/src/hdf5-${HDF5_VERSION}.tar.gz + curl -LO https://support.hdfgroup.org/releases/hdf5/v${VER_MAJOR}/v${VER_NOPATCH}/downloads/hdf5-${HDF5_VERSION}.tar.gz tar -zxf hdf5-${HDF5_VERSION}.tar.gz cd hdf5-${HDF5_VERSION} ./configure --prefix=${GITHUB_WORKSPACE}/HDF5 \ diff --git a/configure.ac b/configure.ac index 18287e64..5762c11a 100644 --- a/configure.ac +++ b/configure.ac @@ -807,14 +807,51 @@ if test "x$have_hdf5" = xyes ; then extra_libs="`grep 'Extra libraries' $hdf5_lib/libhdf5.settings |cut -f2 -d:`" AC_MSG_RESULT([$extra_libs]) fi - if test "x$LIBS" = x ; then - LIBS="-lhdf5 $extra_libs" - else + + # Building HDF5 1.14.6 using cmake produces an incorrect results in the field of + # "Extra libraries". Test it first before including it to LIBS. + if test "x$extra_libs" != x ; then + saved_LIBS=$LIBS LIBS="-lhdf5 $extra_libs $LIBS" + AC_MSG_CHECKING([extra libraries shown in libhdf5.settings if valid]) + AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include ]], + [[H5Fopen("dummy", H5F_ACC_RDWR, H5P_DEFAULT);]])], + [valid_extra_libs=yes], [valid_extra_libs=no]) + AC_MSG_RESULT([$valid_extra_libs]) + if test "x$valid_extra_libs" = xno && test -d "$hdf5_lib/pkgconfig" ; then + # extra_libs is not valid, check pkg-config + saved_PKG_CONFIG_PATH=$PKG_CONFIG_PATH + export PKG_CONFIG_PATH="$hdf5_lib/pkgconfig:$PKG_CONFIG_PATH" + extra_ldflags=`pkg-config --libs-only-L hdf5 2> conftest.err` + rm -f conftest.err + extra_libs=`pkg-config --libs-only-l hdf5 2> conftest.err` + rm -f conftest.err + export PKG_CONFIG_PATH=$saved_PKG_CONFIG_PATH + if test "x$extra_ldflags" != x || test "x$extra_libs" != x ; then + saved_LDFLAGS=$LDFLAGS + LDFLAGS="$extra_libs $LDFLAGS" + LIBS="-lhdf5 $extra_libs $saved_LIBS" + AC_MSG_CHECKING([extra libraries shown in libhdf5.settings if valid]) + AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include ]], + [[H5Fopen("dummy", H5F_ACC_RDWR, H5P_DEFAULT);]])], + [valid_extra_libs=yes], [valid_extra_libs=no]) + AC_MSG_RESULT([$valid_extra_libs]) + if test "x$valid_extra_libs" = xno ; then + extra_ldflags= + extra_libs= + fi + LDFLAGS=$saved_LDFLAGS + fi + fi + LIBS=$saved_LIBS fi - if test "x$LDFLAGS" = x ; then - LDFLAGS=$extra_ldflags - else + + LIBS="$extra_libs $LIBS" + if ! echo $LIBS | grep -q "\-lhdf5" ; then + LIBS="-lhdf5 $LIBS" + fi + LDFLAGS="$extra_ldflags $LDFLAGS" + if ! echo $LDFLAGS | grep -q "\-L$hdf5_lib" ; then LDFLAGS="$extra_ldflags $LDFLAGS" fi AC_DEFINE([ENABLE_HDF5], [1], [Enable HDF5 I/O method])