HDF5 filters (compression) #4489
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: 🐧 Linux | |
on: [push, pull_request] | |
concurrency: | |
group: ${{ github.ref }}-${{ github.head_ref }}-linux | |
cancel-in-progress: true | |
jobs: | |
clang11_nopy_nompi_h5_libcpp: | |
runs-on: ubuntu-22.04 | |
if: github.event.pull_request.draft == false | |
steps: | |
- uses: actions/checkout@v4 | |
- name: Spack Cache | |
uses: actions/cache@v3 | |
with: {path: /opt/spack, key: clang11_nopy_nompi_h5_libcpp_v2} | |
- name: Install | |
run: | | |
sudo apt-get update | |
sudo apt-get install clang-11 libc++-11-dev libc++abi-11-dev | |
sudo .github/workflows/dependencies/install_spack | |
- name: Build | |
env: {CC: clang-11, CXX: clang++-11, CXXFLAGS: -stdlib=libc++ -Werror -Wno-ignored-attributes -Wno-unused-const-variable} | |
# -Wno-ignored-attributes -Wno-unused-const-variable: clang-6 has a | |
# false positive on src/auxiliary/Filesystem.cpp | |
# [[maybe_unused]] MPI_Datatype const MPI_Types< unsigned >::value = MPI_UNSIGNED; | |
run: | | |
eval $(spack env activate --sh .github/ci/spack-envs/clang11_nopy_nompi_h5_libcpp/) | |
spack install | |
share/openPMD/download_samples.sh build | |
cmake -S . -B build \ | |
-DopenPMD_USE_PYTHON=OFF \ | |
-DopenPMD_USE_MPI=OFF \ | |
-DopenPMD_USE_HDF5=ON \ | |
-DopenPMD_USE_INVASIVE_TESTS=ON \ | |
-DCMAKE_VERBOSE_MAKEFILE=ON | |
cmake --build build --parallel 4 | |
cd build | |
ctest --output-on-failure | |
clang11_nopy_ompi_h5_ad2_libcpp: | |
runs-on: ubuntu-22.04 | |
if: github.event.pull_request.draft == false | |
steps: | |
- uses: actions/checkout@v4 | |
- name: Spack Cache | |
uses: actions/cache@v3 | |
with: {path: /opt/spack, key: clang11_nopy_ompi_h5_ad2_libcpp_v2} | |
- name: Install | |
run: | | |
sudo apt-get update | |
sudo apt-get install clang-11 libc++-11-dev libc++abi-11-dev gfortran libopenmpi-dev python3 | |
sudo .github/workflows/dependencies/install_spack | |
- name: Build | |
env: {CC: clang-11, CXX: clang++-11, CXXFLAGS: -stdlib=libc++ -Werror -Wno-ignored-attributes -Wno-unused-const-variable} | |
# -Wno-ignored-attributes -Wno-unused-const-variable: clang-6 has a | |
# false positive on src/auxiliary/Filesystem.cpp | |
# [[maybe_unused]] MPI_Datatype const MPI_Types< unsigned >::value = MPI_UNSIGNED; | |
run: | | |
cmake --version | |
mpiexec --version | |
perl --version | |
python --version | |
sudo ln -s "$(which cmake)" /usr/bin/cmake | |
eval $(spack env activate --sh .github/ci/spack-envs/clang11_nopy_ompi_h5_ad2_libcpp/) | |
spack install | |
share/openPMD/download_samples.sh build | |
cmake -S . -B build \ | |
-DopenPMD_USE_PYTHON=OFF \ | |
-DopenPMD_USE_MPI=ON \ | |
-DopenPMD_USE_HDF5=ON \ | |
-DopenPMD_USE_ADIOS2=ON \ | |
-DopenPMD_USE_INVASIVE_TESTS=ON \ | |
-DCMAKE_VERBOSE_MAKEFILE=ON | |
cmake --build build --parallel 4 | |
cd build | |
ctest --output-on-failure | |
find . -name *.bp | xargs -n1 -P1 -I {} rm -rf {} | |
find . -name *.bp.dir | xargs -n1 -P1 -I {} rm -rf {} | |
ctest --output-on-failure | |
clang11_nopy_ompi_h5_ad2: | |
runs-on: ubuntu-22.04 | |
if: github.event.pull_request.draft == false | |
steps: | |
- uses: actions/checkout@v4 | |
- name: Spack Cache | |
uses: actions/cache@v3 | |
with: {path: /opt/spack, key: clang11_nopy_ompi_h5_ad2_v2 } | |
- name: Install | |
run: | | |
sudo apt-get update | |
sudo apt-get install clang-11 gfortran libopenmpi-dev python3 python3-pip | |
sudo .github/workflows/dependencies/install_spack | |
# Use this to make the HDF5 plugins available from the C/C++ API. | |
export HDF5_PLUGIN_PATH="$(sudo -E .github/workflows/dependencies/install_hdf5_plugins)" | |
echo "$HDF5_PLUGIN_PATH" | |
ls "$HDF5_PLUGIN_PATH" | |
- name: Build | |
env: {CC: clang-11, CXX: clang++-11, CXXFLAGS: -Werror} | |
run: | | |
sudo ln -s "$(which cmake)" /usr/bin/cmake | |
eval $(spack env activate --sh .github/ci/spack-envs/clang11_nopy_ompi_h5_ad2/) | |
spack install | |
share/openPMD/download_samples.sh build | |
cmake -S . -B build \ | |
-DopenPMD_USE_PYTHON=OFF \ | |
-DopenPMD_USE_MPI=ON \ | |
-DopenPMD_USE_HDF5=ON \ | |
-DopenPMD_USE_ADIOS2=ON \ | |
-DopenPMD_USE_INVASIVE_TESTS=ON \ | |
-DCMAKE_VERBOSE_MAKEFILE=ON | |
cmake --build build --parallel 4 | |
cd build | |
ctest --output-on-failure | |
# TODO | |
# clang7_py36_nompi_h5_ad2_libstdc++ | |
clang15_py311_nompi_h5_ad2: | |
runs-on: ubuntu-22.04 | |
if: github.event.pull_request.draft == false | |
steps: | |
- uses: actions/checkout@v4 | |
- name: Spack Cache | |
uses: actions/cache@v3 | |
with: {path: /opt/spack, key: clang15_py311_nompi_h5_ad2_v2 } | |
- name: Install | |
run: | | |
sudo apt update | |
sudo apt install clang-15 cmake gfortran libhdf5-dev python3.11 python3.11-dev wget | |
sudo .github/workflows/dependencies/install_spack | |
python3.11 -m pip install numpy pandas | |
git clone -b v4.0.3 https://github.com/ToruNiina/toml11 | |
cmake -S toml11 -B build_toml11 \ | |
-DCMAKE_INSTALL_PREFIX=toml11_install \ | |
-DCMAKE_CXX_STANDARD_REQUIRED=OFF \ | |
-DCMAKE_CXX_STANDARD=11 | |
cmake --build build_toml11 -j 4 --target install | |
- name: Build | |
env: {CC: clang-15, CXX: clang++-15, CXXFLAGS: -Werror} | |
run: | | |
eval $(spack env activate --sh .github/ci/spack-envs/clang15_py311_nompi_h5_ad2/) | |
spack install | |
share/openPMD/download_samples.sh build | |
export CMAKE_PREFIX_PATH="$(realpath toml11_install):$CMAKE_PREFIX_PATH" | |
cmake -S . -B build \ | |
-DopenPMD_USE_PYTHON=ON \ | |
-DopenPMD_USE_MPI=OFF \ | |
-DopenPMD_USE_HDF5=ON \ | |
-DopenPMD_USE_ADIOS2=ON \ | |
-DopenPMD_USE_INVASIVE_TESTS=ON \ | |
-DopenPMD_USE_INTERNAL_TOML11=OFF \ | |
-DCMAKE_VERBOSE_MAKEFILE=ON \ | |
-DPython_EXECUTABLE="$(which python3.11)" | |
cmake --build build --parallel 4 | |
ctest --test-dir build --output-on-failure | |
# ADIOS2 v2.10 | |
gcc13_py312_mpich_h5_ad2: | |
runs-on: ubuntu-24.04 | |
if: github.event.pull_request.draft == false | |
steps: | |
- uses: actions/checkout@v4 | |
- name: Spack Cache | |
uses: actions/cache@v3 | |
with: {path: /opt/spack, key: gcc13_py312_mpich_h5_ad2 } | |
- name: Install | |
run: | | |
sudo apt-get update | |
sudo apt-get remove openmpi* libopenmpi* *hdf5* || true | |
sudo apt-get install g++ gfortran python3 | |
sudo .github/workflows/dependencies/install_spack | |
# Use this to make the HDF5 plugins available from the C/C++ API. | |
export HDF5_PLUGIN_PATH="$(sudo -E .github/workflows/dependencies/install_hdf5_plugins)" | |
echo "$HDF5_PLUGIN_PATH" | |
ls "$HDF5_PLUGIN_PATH" | |
# Need to build this manually due to broken MPICH package in Ubuntu 24.04 | |
# https://bugs.launchpad.net/ubuntu/+source/mpich/+bug/2072338 | |
sudo .github/workflows/dependencies/install_mpich | |
- name: Build | |
env: {CC: gcc, CXX: g++, MPICH_CC: gcc, MPICH_CXX: g++, CXXFLAGS: -Werror} | |
run: | | |
cmake --version | |
mpiexec --version | |
mpicxx --version | |
perl --version | |
python --version | |
eval $(spack env activate --sh .github/ci/spack-envs/gcc13_py312_mpich_h5_ad2/) | |
spack install | |
share/openPMD/download_samples.sh build | |
cmake -S . -B build \ | |
-DopenPMD_USE_PYTHON=OFF \ | |
-DopenPMD_USE_MPI=ON \ | |
-DopenPMD_USE_HDF5=ON \ | |
-DopenPMD_USE_ADIOS2=ON \ | |
-DopenPMD_USE_INVASIVE_TESTS=ON | |
cmake --build build --parallel 4 | |
cd build | |
ctest --output-on-failure | |
# TODO: (old Travis-CI coverage) | |
# clang10_py38_ompi_h5_1-10-6_ad2_release | |
# ..._h5coll with OPENPMD_HDF5_INDEPENDENT: OFF | |
# TODO: (old Travis-CI coverage) | |
# gcc-4.9.4_nopy_nompi_h5_ad2 | |
# gcc-4.9.4_nopy_ompi_h5_ad2 | |
# gcc-9.3.0_nopy_nompi_h5_ad2-2.6.0 | |
# gcc-7.4.0_nopy_ompi_h5_h5coll | |
# gcc-6.5.0_py35_nompi_h5-1.13.1 | |
# gcc-8.1.0_py37_nompi_h5_static | |
# gcc-6.5.0_py36_nompi_h5-1.8.13 | |
# gcc-4.8.5_py35_nompi_h5 | |
# gcc-7.4.0_py_ompi_h5_ad2_coveralls | |
gcc12_py36_pd_dd_ompi_h5_ad2: | |
runs-on: ubuntu-22.04 | |
if: github.event.pull_request.draft == false | |
steps: | |
- uses: actions/checkout@v4 | |
- name: Spack Cache | |
uses: actions/cache@v3 | |
with: {path: /opt/spack, key: gcc12_py36_ompi_h5_ad2_v2 } | |
- name: Install | |
run: | | |
sudo apt-get update | |
sudo apt-get install g++-12 gfortran libopenmpi-dev python3 python3-setuptools | |
sudo .github/workflows/dependencies/install_spack | |
# Use this to make the HDF5 plugins available from the C/C++ API. | |
export HDF5_PLUGIN_PATH="$(sudo -E .github/workflows/dependencies/install_hdf5_plugins)" | |
echo "$HDF5_PLUGIN_PATH" | |
ls "$HDF5_PLUGIN_PATH" | |
python3 -m pip install -U pip | |
python3 -m pip install -U numpy | |
python3 -m pip install -U mpi4py | |
python3 -m pip install -U pandas | |
python3 -m pip install -U dask | |
python3 -m pip install -U pyarrow | |
- name: Build | |
env: {CC: gcc-12, CXX: g++-12, CXXFLAGS: -Werror} | |
run: | | |
sudo ln -s "$(which cmake)" /usr/bin/cmake | |
eval $(spack env activate --sh .github/ci/spack-envs/gcc12_py36_ompi_h5_ad2/) | |
spack install | |
share/openPMD/download_samples.sh build | |
cmake -S . -B build \ | |
-DopenPMD_USE_PYTHON=ON \ | |
-DopenPMD_USE_MPI=ON \ | |
-DopenPMD_USE_HDF5=ON \ | |
-DopenPMD_USE_ADIOS2=ON \ | |
-DopenPMD_USE_INVASIVE_TESTS=ON \ | |
-DCMAKE_VERBOSE_MAKEFILE=ON | |
cmake --build build --parallel 4 | |
ctest --test-dir build --output-on-failure | |
gcc9_py38_pd_nompi_h5_ad2_libcpp: | |
runs-on: ubuntu-22.04 | |
if: github.event.pull_request.draft == false | |
steps: | |
- uses: actions/checkout@v4 | |
- name: Install | |
run: | | |
sudo apt-get update | |
sudo apt-get install g++ libopenmpi-dev libhdf5-openmpi-dev python3 python3-numpy python3-mpi4py python3-pandas python3-h5py-mpi python3-pip | |
python3 -m pip install jsonschema==4.* referencing hdf5plugin | |
# TODO ADIOS2 | |
- name: Build | |
env: {CXXFLAGS: -Werror, PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig} | |
run: | | |
share/openPMD/download_samples.sh build | |
# Run a debug build, | |
# Pybind has some builtin checks only activated in debug mode. | |
cmake -S . -B build \ | |
-DopenPMD_USE_PYTHON=ON \ | |
-DopenPMD_USE_MPI=ON \ | |
-DopenPMD_USE_HDF5=ON \ | |
-DopenPMD_USE_FILESYSTEM_HEADER=ON \ | |
-DopenPMD_USE_INVASIVE_TESTS=ON \ | |
-DCMAKE_BUILD_TYPE=Debug | |
cmake --build build --parallel 4 | |
ctest --test-dir build --output-on-failure | |
cd share/openPMD/json_schema | |
PATH="../../../build/bin:$PATH" make -j 2 | |
# We need to exclude the thetaMode example since that has a different | |
# meshesPath and the JSON schema needs to hardcode that. | |
find ../../../build/samples/ \ | |
! -path '*thetaMode*' \ | |
! -path '/*many_iterations/*' \ | |
! -name 'profiling.json' \ | |
! -name '*config.json' \ | |
-iname '*.json' \ | |
| while read i; do | |
echo "Checking $i" | |
./check.py "$i" | |
done | |
musllinux_py10: | |
runs-on: ubuntu-22.04 | |
if: github.event.pull_request.draft == false | |
container: | |
image: quay.io/pypa/musllinux_1_1_x86_64 | |
steps: | |
- uses: actions/checkout@v4 | |
- name: Install | |
run: | | |
apk update | |
apk add hdf5-dev | |
python3.10 -m pip install numpy h5py hdf5plugin | |
- name: Build | |
env: {CXXFLAGS: -Werror} | |
run: | | |
share/openPMD/download_samples.sh build | |
cmake -S . -B build \ | |
-DopenPMD_USE_PYTHON=ON \ | |
-DopenPMD_USE_MPI=OFF \ | |
-DopenPMD_USE_HDF5=ON \ | |
-DopenPMD_USE_INVASIVE_TESTS=ON \ | |
-DopenPMD_USE_FILESYSTEM_HEADER=ON \ | |
-DPython_EXECUTABLE=$(which python3.10) | |
cmake --build build --parallel 4 | |
ctest --test-dir build --output-on-failure | |
conda_ompi_all: | |
runs-on: ubuntu-22.04 | |
if: github.event.pull_request.draft == false | |
steps: | |
- uses: actions/checkout@v4 | |
- uses: conda-incubator/setup-miniconda@v3 | |
name: Setup conda | |
with: | |
auto-update-conda: true | |
activate-environment: openpmd-api-dev | |
environment-file: conda.yml | |
channels: conda-forge,defaults | |
channel-priority: true | |
- name: Build | |
shell: bash -eo pipefail -l {0} | |
env: {CXXFLAGS: -Werror} | |
run: | | |
share/openPMD/download_samples.sh build | |
cmake -S . -B build \ | |
-DCMAKE_CXX_FLAGS="-Wno-error=stringop-overread" \ | |
-DCMAKE_C_FLAGS="-Wno-error=stringop-overread" \ | |
-DopenPMD_USE_PYTHON=ON \ | |
-DopenPMD_USE_MPI=ON \ | |
-DopenPMD_USE_HDF5=ON \ | |
-DopenPMD_USE_ADIOS2=ON \ | |
-DopenPMD_USE_FILESYSTEM_HEADER=ON \ | |
-DopenPMD_USE_INVASIVE_TESTS=ON | |
cmake --build build --parallel 4 | |
ctest --test-dir build --output-on-failure |