ADIOS2: Require v2.9 as minimum (#1711)

* CMake: Require ADIOS2 v2.9

* Remove openPMD_HAS_ADIOS_2_8 macro

* Remove openPMD_HAS_ADIOS_2_9 macro

* Fixes

* Update ADIOS2 in the CI

* Only ADIOS2 v2.9 on Spack?

* Revert "Only ADIOS2 v2.9 on Spack?"

This reverts commit 1740136.

* Update spack

* Only specify major ADIOS2 versions

* Fixes: Mgard, /usr/bin/cmake

* Try ADIOS2 v2.10 for failing runs

* Take mgard out of clang7_nopy_ompi_h5_ad2_libcpp run

* Take blosc2 out of failing runs

* Update documentation

* Reimplement "modifiable_attributes" parameter

* Revert "Reimplement "modifiable_attributes" parameter"

This reverts commit f3efc99.

Author: franzpoeschel

Diff for: .github/ci/spack-envs/clang14_py311_nompi_h5_ad2/spack.yaml

@@ -13,7 +13,7 @@ spack:
     hdf5:
       variants: ~mpi
     adios2:
-      variants: ~mpi ~zfp ~sz ~png ~dataman ~python ~fortran ~ssc ~shared ~bzip2
+      variants: ~mpi ~zfp ~sz ~png ~dataman ~python ~fortran ~ssc ~shared ~bzip2 ~mgard
     cmake:
       externals:
       - spec: [email protected]

Diff for: .github/ci/spack-envs/clang7_nopy_ompi_h5_ad2/spack.yaml

@@ -12,7 +12,7 @@ spack:
 
   packages:
     adios2:
-      variants: ~zfp ~sz ~png ~dataman ~python ~fortran ~ssc ~shared ~bzip2
+      variants: ~zfp ~sz ~png ~dataman ~python ~fortran ~ssc ~shared ~bzip2 ~blosc2
     cmake:
       externals:
       - spec: [email protected]

Diff for: .github/ci/spack-envs/clang7_nopy_ompi_h5_ad2_libcpp/spack.yaml

@@ -6,13 +6,13 @@
 #
 spack:
   specs:
-  - adios2
+  - adios2@2.10
   - hdf5
   - openmpi
 
   packages:
     adios2:
-      variants: ~zfp ~sz ~png ~dataman ~python ~fortran ~ssc ~shared ~bzip2
+      variants: ~zfp ~sz ~png ~dataman ~python ~fortran ~ssc ~shared ~bzip2 ~mgard ~blosc2
     cmake:
       externals:
       - spec: [email protected]

Diff for: .github/ci/spack-envs/clang8_py38_mpich_h5_ad2/spack.yaml

@@ -6,13 +6,13 @@
 #
 spack:
   specs:
-  - adios2@2.7.1
+  - adios2@2.10
   - hdf5
   - mpich
 
   packages:
     adios2:
-      variants: ~zfp ~sz ~png ~dataman ~python ~fortran ~ssc ~shared ~bzip2
+      variants: ~zfp ~sz ~png ~dataman ~python ~fortran ~ssc ~shared ~bzip2 ~blosc2
     cmake:
       externals:
       - spec: [email protected]

Diff for: .github/ci/spack-envs/clangtidy_nopy_ompi_h5_ad2/spack.yaml

@@ -6,7 +6,7 @@
 #
 spack:
   specs:
-  - adios2
+  - adios2@2.10
   - hdf5
   - openmpi
 

Diff for: .github/ci/spack-envs/gcc7_py36_ompi_h5_ad2/spack.yaml

@@ -6,7 +6,7 @@
 #
 spack:
   specs:
-  - adios2
+  - adios2@2.10
   - hdf5
   - openmpi
 

Diff for: .github/workflows/dependencies/install_spack

@@ -3,7 +3,7 @@
 
 set -eu -o pipefail
 
-spack_ver="0.17.1"
+spack_ver="0.22.3"
 
 cd /opt
 if [[ -d spack && ! -f spack_${spack_ver} ]]

Diff for: .github/workflows/linux.yml

@@ -63,6 +63,7 @@ jobs:
         mpiexec --version
         perl --version
         python --version
+        sudo ln -s "$(which cmake)" /usr/bin/cmake
         eval $(spack env activate --sh .github/ci/spack-envs/clang7_nopy_ompi_h5_ad2_libcpp/)
         spack install
 
@@ -99,6 +100,7 @@ jobs:
     - name: Build
       env: {CC: clang-7, CXX: clang++-7, CXXFLAGS: -Werror}
       run: |
+        sudo ln -s "$(which cmake)" /usr/bin/cmake
         eval $(spack env activate --sh .github/ci/spack-envs/clang7_nopy_ompi_h5_ad2/)
         spack install
 
@@ -178,6 +180,7 @@ jobs:
         mpiexec --version
         perl --version
         python --version
+        sudo ln -s "$(which cmake)" /usr/bin/cmake
         eval $(spack env activate --sh .github/ci/spack-envs/clang8_py38_mpich_h5_ad2/)
         spack install
 
@@ -229,6 +232,7 @@ jobs:
     - name: Build
       env: {CC: gcc-7, CXX: g++-7, CXXFLAGS: -Werror}
       run: |
+        sudo ln -s "$(which cmake)" /usr/bin/cmake
         eval $(spack env activate --sh .github/ci/spack-envs/gcc7_py36_ompi_h5_ad2/)
         spack install
 

Diff for: .github/workflows/tooling.yml

@@ -24,6 +24,7 @@ jobs:
     - name: Build
       env: {CC: clang, CXX: clang++}
       run: |
+        sudo ln -s "$(which cmake)" /usr/bin/cmake
         eval $(spack env activate --sh .github/ci/spack-envs/clangtidy_nopy_ompi_h5_ad2/)
         spack install
 
@@ -54,6 +55,7 @@ jobs:
     - name: Build
       env: {CC: mpicc, CXX: mpic++, OMPI_CC: clang-10, OMPI_CXX: clang++-10, CXXFLAGS: -Werror, OPENPMD_HDF5_CHUNKS: none, OPENPMD_TEST_NFILES_MAX: 100}
       run: |
+        sudo ln -s "$(which cmake)" /usr/bin/cmake
         eval $(spack env activate --sh .github/ci/spack-envs/clangtidy_nopy_ompi_h5_ad2/)
         spack install
         SOURCEPATH="$(pwd)"

Diff for: CHANGELOG.rst

@@ -3,6 +3,17 @@
 Changelog
 =========
 
+0.16.0
+------
+
+Changes to "0.15.0"
+^^^^^^^^^^^^^^^^^^^
+
+Other
+"""""
+
+- ADIOS2: require version 2.9.0 #1711
+
 0.16.0
 ------
 **Date:** 2024-10-07

Diff for: CMakeLists.txt

@@ -365,14 +365,14 @@ if(openPMD_HAVE_MPI)
     list(APPEND openPMD_REQUIRED_ADIOS2_COMPONENTS MPI)
 endif()
 if(openPMD_USE_ADIOS2 STREQUAL AUTO)
-    find_package(ADIOS2 2.7.0 CONFIG COMPONENTS ${openPMD_REQUIRED_ADIOS2_COMPONENTS})
+    find_package(ADIOS2 2.9.0 CONFIG COMPONENTS ${openPMD_REQUIRED_ADIOS2_COMPONENTS})
     if(ADIOS2_FOUND)
         set(openPMD_HAVE_ADIOS2 TRUE)
     else()
         set(openPMD_HAVE_ADIOS2 FALSE)
     endif()
 elseif(openPMD_USE_ADIOS2)
-    find_package(ADIOS2 2.7.0 REQUIRED CONFIG COMPONENTS ${openPMD_REQUIRED_ADIOS2_COMPONENTS})
+    find_package(ADIOS2 2.9.0 REQUIRED CONFIG COMPONENTS ${openPMD_REQUIRED_ADIOS2_COMPONENTS})
     set(openPMD_HAVE_ADIOS2 TRUE)
 else()
     set(openPMD_HAVE_ADIOS2 FALSE)

Diff for: Dockerfile

@@ -56,7 +56,7 @@ RUN        curl -sLo c-blosc-1.15.0.tar.gz https://github.com/Blosc/c-blosc/arch
            && make \
            && make install
 
-RUN        curl -sLo adios2-2.7.1.tar.gz https://github.com/ornladios/ADIOS2/archive/v2.7.1.tar.gz \
+RUN        curl -sLo adios2-2.9.2.tar.gz https://github.com/ornladios/ADIOS2/archive/v2.9.2.tar.gz \
            && file adios2*.tar.gz \
            && tar -xzf adios2*.tar.gz \
            && rm adios2*.tar.gz \

Diff for: NEWS.rst

@@ -3,6 +3,11 @@
 Upgrade Guide
 =============
 
+0.17.0
+------
+
+ADIOS 2.9.0 is now the minimally supported version for ADIOS2 support.
+
 0.16.0
 ------
 

Diff for: README.md

@@ -108,7 +108,7 @@ Shipped internally (downloaded by CMake unless `openPMD_SUPERBUILD=OFF` is set):
 I/O backends:
 * [JSON](https://en.wikipedia.org/wiki/JSON)
 * [HDF5](https://support.hdfgroup.org/HDF5) 1.8.13+ (optional)
-* [ADIOS2](https://github.com/ornladios/ADIOS2) 2.7.0+ (optional)
+* [ADIOS2](https://github.com/ornladios/ADIOS2) 2.9.0+ (optional)
 
 while those can be built either with or without:
 * MPI 2.1+, e.g. OpenMPI 1.6.5+ or MPICH2

Diff for: docs/source/backends/adios2.rst

@@ -58,9 +58,6 @@ Steps
 ADIOS2 is optimized towards organizing the process of reading/writing data into IO steps.
 In order to activate steps, it is imperative to use the :ref:`Streaming API <usage-streaming>` (which can be used for either file-based or streaming-based workflows).
 
-ADIOS2 release 2.6.0 contained a bug (fixed in ADIOS 2.7.0, see `PR #2348 <https://github.com/ornladios/ADIOS2/pull/2348>`_) that disallows random-accessing steps in file-based engines.
-With this ADIOS2 release, files written with steps may only be read using the streaming API.
-
 Upon reading a file, the ADIOS2 backend will automatically recognize whether it has been written with or without steps, ignoring the JSON option mentioned above.
 Steps are mandatory for streaming-based engines and trying to switch them off will result in a runtime error.
 

Diff for: docs/source/dev/dependencies.rst

@@ -27,7 +27,7 @@ Optional: I/O backends
 
 * `JSON <https://en.wikipedia.org/wiki/JSON>`_
 * `HDF5 <https://support.hdfgroup.org/HDF5>`_ 1.8.13+
-* `ADIOS2 <https://github.com/ornladios/ADIOS2>`_ 2.7.0+
+* `ADIOS2 <https://github.com/ornladios/ADIOS2>`_ 2.9.0+
 
 while those can be build either with or without:
 

Diff for: include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp

@@ -198,9 +198,7 @@ namespace detail
         case adios2::Mode::Write:
             return false;
         case adios2::Mode::Read:
-#if openPMD_HAS_ADIOS_2_8
         case adios2::Mode::ReadRandomAccess:
-#endif
             return true;
         case adios2::Mode::Undefined:
         case adios2::Mode::Sync:
@@ -217,9 +215,7 @@ namespace detail
         case adios2::Mode::Write:
             return true;
         case adios2::Mode::Read:
-#if openPMD_HAS_ADIOS_2_8
         case adios2::Mode::ReadRandomAccess:
-#endif
             return false;
         case adios2::Mode::Undefined:
         case adios2::Mode::Sync:

Diff for: include/openPMD/IO/ADIOS/macros.hpp

@@ -6,19 +6,6 @@
 
 #include <adios2.h>
 
-/*
- * ADIOS2 v2.8 brings mode::ReadRandomAccess
- */
-#define openPMD_HAS_ADIOS_2_8                                                  \
-    (ADIOS2_VERSION_MAJOR * 100 + ADIOS2_VERSION_MINOR >= 208)
-/*
- * ADIOS2 v2.9 brings modifiable attributes (technically already in v2.8, but
- * there are too many bugs, so we only support it beginning with v2.9).
- * Group table feature requires ADIOS2 v2.9.
- */
-#define openPMD_HAS_ADIOS_2_9                                                  \
-    (ADIOS2_VERSION_MAJOR * 100 + ADIOS2_VERSION_MINOR >= 209)
-
 #define openPMD_HAS_ADIOS_2_10                                                 \
     (ADIOS2_VERSION_MAJOR * 100 + ADIOS2_VERSION_MINOR >= 210)
 

Diff for: src/IO/ADIOS/ADIOS2File.cpp

@@ -428,8 +428,6 @@ void ADIOS2File::configure_IO()
     // these properties are inferred from the opened dataset in read mode
     if (writeOnly(m_mode))
     {
-
-#if openPMD_HAS_ADIOS_2_9
         if (!m_impl->m_useGroupTable.has_value())
         {
             switch (m_impl->m_handler->m_encoding)
@@ -456,15 +454,6 @@ void ADIOS2File::configure_IO()
                 ? ADIOS2IOHandlerImpl::ModifiableAttributes::Yes
                 : ADIOS2IOHandlerImpl::ModifiableAttributes::No;
         }
-#else
-        if (!m_impl->m_useGroupTable.has_value())
-        {
-            m_impl->m_useGroupTable = UseGroupTable::No;
-        }
-
-        m_impl->m_modifiableAttributes =
-            ADIOS2IOHandlerImpl::ModifiableAttributes::No;
-#endif
     }
 
     // set engine type
@@ -764,9 +753,7 @@ adios2::Engine &ADIOS2File::getEngine()
             streamStatus = StreamStatus::DuringStep;
             break;
         }
-#if openPMD_HAS_ADIOS_2_8
         case adios2::Mode::ReadRandomAccess:
-#endif
         case adios2::Mode::Read: {
             m_engine =
                 std::make_optional(adios2::Engine(m_IO.Open(m_file, m_mode)));
@@ -1320,9 +1307,7 @@ void ADIOS2File::markActive(Writable *writable)
     {
     case UseGroupTable::No:
         break;
-    case UseGroupTable::Yes:
-#if openPMD_HAS_ADIOS_2_9
-    {
+    case UseGroupTable::Yes: {
         if (writeOnly(m_mode) && m_impl->m_writeAttributesFromThisRank)
         {
             auto currentStepBuffered = currentStep();
@@ -1346,12 +1331,6 @@ void ADIOS2File::markActive(Writable *writable)
                          m_pathsMarkedAsActive.end());
         }
     }
-#else
-        (void)writable;
-        throw error::OperationUnsupportedInBackend(
-            m_impl->m_handler->backendName(),
-            "Group table feature requires ADIOS2 >= v2.9.");
-#endif
     break;
     }
 }