openPMD Output

.github/workflows/source/doxygen

@@ -15,6 +15,9 @@ curl -L -o amrex-doxygen-web.tag.xml \
 curl -L -o warpx-doxygen-web.tag.xml \
   https://warpx.readthedocs.io/en/latest/_static/doxyhtml/warpx-doxygen-web.tag.xml
 
+curl -L -o openpmd-api-doxygen-web.tag.xml \
+  https://openpmd-api.readthedocs.io/en/latest/_static/doxyhtml/openpmd-api-doxygen-web.tag.xml
+
 # treat all warnings as errors
 echo "WARN_AS_ERROR = YES" >> Doxyfile
 

.github/workflows/ubuntu.yml

@@ -53,11 +53,11 @@ jobs:
       run: |
         sudo cmake --build build --target install
 
-        impactx.MPI.OMP.DP.DEBUG examples/fodo/input_fodo.in algo.particle_shape = 1 || \
+        impactx.MPI.OMP.DP.OPMD.DEBUG examples/fodo/input_fodo.in algo.particle_shape = 1 || \
         { cat Backtrace.0.0; exit 1; }
-        impactx.MPI.OMP.DP.DEBUG examples/fodo/input_fodo.in algo.particle_shape = 2 || \
+        impactx.MPI.OMP.DP.OPMD.DEBUG examples/fodo/input_fodo.in algo.particle_shape = 2 || \
         { cat Backtrace.0.0; exit 1; }
-        impactx.MPI.OMP.DP.DEBUG examples/fodo/input_fodo.in algo.particle_shape = 3 || \
+        impactx.MPI.OMP.DP.OPMD.DEBUG examples/fodo/input_fodo.in algo.particle_shape = 3 || \
         { cat Backtrace.0.0; exit 1; }
 
     - name: run installed python module
@@ -113,11 +113,11 @@ jobs:
       run: |
         sudo cmake --build build --target install
 
-        impactx.NOMPI.OMP.DP.DEBUG examples/fodo/input_fodo.in algo.particle_shape = 1 || \
+        impactx.NOMPI.OMP.DP.OPMD.DEBUG examples/fodo/input_fodo.in algo.particle_shape = 1 || \
         { cat Backtrace.0.0; exit 1; }
-        impactx.NOMPI.OMP.DP.DEBUG examples/fodo/input_fodo.in algo.particle_shape = 2 || \
+        impactx.NOMPI.OMP.DP.OPMD.DEBUG examples/fodo/input_fodo.in algo.particle_shape = 2 || \
         { cat Backtrace.0.0; exit 1; }
-        impactx.NOMPI.OMP.DP.DEBUG examples/fodo/input_fodo.in algo.particle_shape = 3 || \
+        impactx.NOMPI.OMP.DP.OPMD.DEBUG examples/fodo/input_fodo.in algo.particle_shape = 3 || \
         { cat Backtrace.0.0; exit 1; }
 
     - name: run installed python module

.github/workflows/windows.yml

@@ -29,29 +29,44 @@ jobs:
         restore-keys: |
           ccache-windows-winmsvc-${{ hashFiles('.github/workflows/windows.yml') }}-
           ccache-windows-winmsvc-
-    - name: install dependencies
+    - name: Download Dependencies
       run: |
         Invoke-WebRequest https://github.com/HDFGroup/hdf5/archive/refs/tags/hdf5-1_12_2.tar.gz -OutFile hdf5-1_12_2.tar.gz
         7z.exe x -r hdf5-1_12_2.tar.gz
         7z.exe x -r hdf5-1_12_2.tar
-        cmake -S hdf5-hdf5-1_12_2 -B build_hdf5 -DCMAKE_BUILD_TYPE=RelWithDebInfo -DBUILD_TESTING=OFF -DHDF5_BUILD_EXAMPLES=OFF
+    - name: Install Dependencies
+      run: |
+        cmake -S hdf5-hdf5-1_12_2 -B build_hdf5  `
+              -DCMAKE_BUILD_TYPE=RelWithDebInfo  `
+              -DBUILD_SHARED_LIBS=OFF            `
+              -DBUILD_TESTING=OFF                `
+              -DHDF5_BUILD_HL_LIB=OFF            `
+              -DHDF5_BUILD_TOOLS=OFF             `
+              -DHDF5_BUILD_EXAMPLES=OFF          `
+              -DHDF5_ENABLE_SZIP_SUPPORT=OFF     `
+              -DHDF5_ENABLE_Z_LIB_SUPPORT=OFF
         if(!$?) { Exit $LASTEXITCODE }
         cmake --build build_hdf5 --config RelWithDebInfo --target install --parallel 2
         if(!$?) { Exit $LASTEXITCODE }
-    - name: Build
-      run: |
+
+        $env:HDF5_DIR = "C:/Program Files/HDF_Group/HDF5/1.12.2/cmake/"
+        $env:HDF5_USE_STATIC_LIBRARIES = "ON"
+
         python3 -m pip install -U pip setuptools wheel pytest
         python3 -m pip install -r requirements.txt
         python3 -m pip install -r examples/requirements.txt
-
-        $env:HDF5_DIR = "C:/Program Files (x86)/hdf5/"
+    - name: Build
+      run: |
+        $env:HDF5_DIR = "C:/Program Files/HDF_Group/HDF5/1.12.2/cmake/"
         cmake -S . -B build               `
               -DBUILD_SHARED_LIBS=ON      `
               -DCMAKE_BUILD_TYPE=RelWithDebInfo  `
-              -DCMAKE_VERBOSE_MAKEFILE=ON `
+              -DCMAKE_VERBOSE_MAKEFILE=ON    `
+              -DHDF5_USE_STATIC_LIBRARIES=ON `
               -DImpactX_COMPUTE=NOACC     `
               -DImpactX_MPI=OFF           `
               -DImpactX_PYTHON=ON         `
+              -DopenPMD_USE_HDF5=ON       `
               -DPython_EXECUTABLE=python3
         if(!$?) { Exit $LASTEXITCODE }
         cmake --build build --config RelWithDebInfo --parallel 2
@@ -90,50 +105,73 @@ jobs:
           ccache-windows-winclang-${{ hashFiles('.github/workflows/windows.yml') }}-
           ccache-windows-winclang-
     - uses: seanmiddleditch/gha-setup-ninja@master
-    - name: install dependencies
+    - name: Download Dependencies
       run: |
         Invoke-WebRequest https://github.com/HDFGroup/hdf5/archive/refs/tags/hdf5-1_12_2.tar.gz -OutFile hdf5-1_12_2.tar.gz
         7z.exe x -r hdf5-1_12_2.tar.gz
         7z.exe x -r hdf5-1_12_2.tar
-        cmake -S hdf5-hdf5-1_12_2 -B build_hdf5 -DCMAKE_BUILD_TYPE=RelWithDebInfo -DBUILD_TESTING=OFF -DHDF5_BUILD_EXAMPLES=OFF
-        if(!$?) { Exit $LASTEXITCODE }
-        cmake --build build_hdf5 --config RelWithDebInfo --target install --parallel 2
-        if(!$?) { Exit $LASTEXITCODE }
-    - name: Build
+    - name: Install Dependencies
       shell: cmd
       run: |
+        call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\vc\Auxiliary\build\vcvarsall.bat" x64
+
+        cmake -S hdf5-hdf5-1_12_2 -B build_hdf5 ^
+            -G "Ninja"                          ^
+            -DCMAKE_C_COMPILER=clang-cl         ^
+            -DCMAKE_CXX_COMPILER=clang-cl       ^
+            -DCMAKE_BUILD_TYPE=Release          ^
+            -DBUILD_SHARED_LIBS=OFF             ^
+            -DBUILD_TESTING=OFF                 ^
+            -DHDF5_BUILD_HL_LIB=OFF             ^
+            -DHDF5_BUILD_TOOLS=OFF              ^
+            -DHDF5_BUILD_EXAMPLES=OFF           ^
+            -DHDF5_ENABLE_SZIP_SUPPORT=OFF      ^
+            -DHDF5_ENABLE_Z_LIB_SUPPORT=OFF
+        if errorlevel 1 exit 1
+        cmake --build build_hdf5 --config Release --target install --parallel 2
+        if errorlevel 1 exit 1
+
+        set "HDF5_DIR=C:/Program Files/HDF_Group/HDF5/1.12.2/cmake/"
+        set "HDF5_USE_STATIC_LIBRARIES=ON"
+
         python3 -m pip install -U pip setuptools wheel pytest
         python3 -m pip install -r requirements.txt
         python3 -m pip install -r examples/requirements.txt
-
+    - name: Build
+      shell: cmd
+      run: |
         call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\vc\Auxiliary\build\vcvarsall.bat" x64
 
-        set "HDF5_DIR=C:/Program Files (x86)/hdf5/"
+        set "HDF5_DIR=C:/Program Files/HDF_Group/HDF5/1.12.2/cmake/"
         cmake -S . -B build   ^
               -G "Ninja"      ^
-              -DCMAKE_C_COMPILER=clang-cl   ^
-              -DCMAKE_CXX_COMPILER=clang-cl ^
-              -DBUILD_SHARED_LIBS=OFF       ^
-              -DCMAKE_BUILD_TYPE=Release    ^
-              -DCMAKE_VERBOSE_MAKEFILE=ON   ^
-              -DImpactX_COMPUTE=OMP         ^
-              -DImpactX_MPI=OFF             ^
-              -DImpactX_PYTHON=ON           ^
+              -DCMAKE_C_COMPILER=clang-cl    ^
+              -DCMAKE_CXX_COMPILER=clang-cl  ^
+              -DBUILD_SHARED_LIBS=OFF        ^
+              -DCMAKE_BUILD_TYPE=Release     ^
+              -DCMAKE_VERBOSE_MAKEFILE=ON    ^
+              -DHDF5_USE_STATIC_LIBRARIES=ON ^
+              -DImpactX_COMPUTE=OMP          ^
+              -DImpactX_MPI=OFF              ^
+              -DImpactX_PYTHON=ON            ^
+              -DopenPMD_USE_HDF5=ON          ^
               -DPython_EXECUTABLE=python3
         if errorlevel 1 exit 1
-        cmake --build build --config Release --parallel 2
+        cmake --build build --parallel 2
         if errorlevel 1 exit 1
     - name: Test
       shell: cmd
       run: |
         call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\vc\Auxiliary\build\vcvarsall.bat" x64
-        ctest --test-dir build --build-config Release --output-on-failure -E AMReX
-        if errorlevel 1 exit 1
-    - name: Install
-      shell: cmd
-      run: |
-        call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\vc\Auxiliary\build\vcvarsall.bat" x64
-        cmake --build build --config Release --target install
-        if errorlevel 1 exit 1
-        cmake --build build --config Release --target pip_install
+        ctest --test-dir build --output-on-failure -E AMReX
         if errorlevel 1 exit 1
+#    - name: Install
+#      shell: cmd
+#      run: |
+#        call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\vc\Auxiliary\build\vcvarsall.bat" x64
+#        set "HDF5_DIR=C:/Program Files/HDF_Group/HDF5/1.12.2/cmake/"
+#
+#        cmake --build build --target install
+#        if errorlevel 1 exit 1
+#        cmake --build build --target pip_install
+#        if errorlevel 1 exit 1

CMakeLists.txt

@@ -56,7 +56,7 @@ include(CMakeDependentOption)
 option(ImpactX_APP           "Build the ImpactX executable application"     ON)
 option(ImpactX_LIB           "Build ImpactX as a library"                   OFF)
 option(ImpactX_MPI           "Multi-node support (message-passing)"         ON)
-option(ImpactX_OPENPMD       "openPMD I/O (HDF5, ADIOS)"                    OFF)
+option(ImpactX_OPENPMD       "openPMD I/O (HDF5, ADIOS)"                    ON)
 option(ImpactX_PYTHON        "Python bindings"                              OFF)
 
 set(ImpactX_PRECISION_VALUES SINGLE DOUBLE)
@@ -78,6 +78,7 @@ mark_as_advanced(ImpactX_MPI_THREAD_MULTIPLE)
 
 option(ImpactX_ablastr_internal                  "Download & build ABLASTR" ON)
 option(ImpactX_amrex_internal                    "Download & build AMReX" ON)
+option(ImpactX_openpmd_internal                  "Download & build openPMD-api" ON)
 option(ImpactX_pyamrex_internal                  "Download & build pyAMReX" ON)
 
 set(pyImpactX_VERSION_INFO "" CACHE STRING
@@ -250,7 +251,6 @@ if(ImpactX_PYTHON)
 endif()
 
 if(ImpactX_OPENPMD)
-    target_compile_definitions(ImpactX PUBLIC ImpactX_USE_OPENPMD)
     target_link_libraries(ImpactX PUBLIC openPMD::openPMD)
 endif()
 

README.md

@@ -208,7 +208,7 @@ In new terminals, re-activate the environment with `spack env activate impactx-d
 ### Conda (Linux/macOS/Windows)
 
 ```bash
-conda create -n impactx-dev -c conda-forge adios2 ccache cmake compilers git hdf5 fftw matplotlib ninja numpy pandas pytest scipy
+conda create -n impactx-dev -c conda-forge adios2 ccache cmake compilers git hdf5 fftw matplotlib ninja numpy openpmd-api pandas pytest scipy
 conda activate impactx-dev
 
 # compile with -DImpactX_MPI=OFF

cmake/dependencies/ABLASTR.cmake

@@ -26,7 +26,28 @@ macro(find_ablastr)
         endif()
     endif()
 
-    # transitive control for openPMD/FFT/PICSAR superbuild
+    # transitive control for openPMD superbuild
+    if(ImpactX_openpmd_src)
+        set(WarpX_openpmd_src ${ImpactX_openpmd_src} CACHE PATH
+            "Local path to openPMD-api source directory (preferred if set)" FORCE)
+    elseif(ImpactX_openpmd_internal)
+        if(ImpactX_openpmd_repo)
+            set(WarpX_openpmd_repo ${ImpactX_openpmd_repo} CACHE STRING
+                "Repository URI to pull and build openPMD-api from if(ImpactX_openpmd_internal)" FORCE)
+        endif()
+        if(ImpactX_openpmd_branch)
+            set(WarpX_openpmd_branch ${ImpactX_openpmd_branch} CACHE STRING
+                "Repository branch for ImpactX_openpmd_repo if(ImpactX_openpmd_internal)" FORCE)
+        endif()
+    else()
+        set(WarpX_openpmd_internal ${ImpactX_openpmd_internal} CACHE STRING
+            "Download & build openPMD-api" FORCE)
+    endif()
+    #   temporary: can be removed once we update to ABLASTR 23.04+
+    set(openPMD_BUILD_SHARED_LIBS  ${BUILD_SHARED_LIBS} CACHE INTERNAL "")
+    set(openPMD_INSTALL  ${openPMD_BUILD_SHARED_LIBS} CACHE INTERNAL "")
+
+    # transitive control for FFT/PICSAR superbuild
     # TODO (future)
 
     # ABLASTR superbuild
@@ -120,6 +141,17 @@ macro(find_ablastr)
         find_package(ABLASTR 23.02 CONFIG REQUIRED COMPONENTS ${COMPONENT_DIM})
         message(STATUS "ABLASTR: Found version '${ABLASTR_VERSION}'")
     endif()
+
+    # transitive control for openPMD external
+    if(NOT ImpactX_openpmd_src AND NOT ImpactX_openpmd_internal)
+        if(ImpactX_MPI)
+            set(COMPONENT_WMPI MPI)
+        else()
+            set(COMPONENT_WMPI NOMPI)
+        endif()
+        find_package(openPMD 0.15.0 CONFIG REQUIRED COMPONENTS ${COMPONENT_WMPI})
+        message(STATUS "openPMD-api: Found version '${openPMD_VERSION}'")
+    endif()
 endmacro()
 
 # local source-tree
@@ -129,12 +161,15 @@ set(ImpactX_amrex_src ""
 set(ImpactX_ablastr_src ""
     CACHE PATH
     "Local path to ABLASTR source directory (preferred if set)")
+set(ImpactX_openpmd_src ""
+    CACHE PATH
+    "Local path to openPMD-api source directory (preferred if set)")
 
 # Git fetcher
 set(ImpactX_ablastr_repo "https://github.com/ECP-WarpX/WarpX.git"
     CACHE STRING
     "Repository URI to pull and build ABLASTR from if(ImpactX_ablastr_internal)")
-set(ImpactX_ablastr_branch "23.02"
+set(ImpactX_ablastr_branch "12be0f989c912dd17fb53640e1006ea33d170ea4"
     CACHE STRING
     "Repository branch for ImpactX_ablastr_repo if(ImpactX_ablastr_internal)")
 
@@ -146,4 +181,12 @@ set(ImpactX_amrex_branch ""
     CACHE STRING
     "Repository branch for ImpactX_amrex_repo if(ImpactX_amrex_internal)")
 
+# openPMD is transitively pulled through ABLASTR
+set(ImpactX_openpmd_repo "https://github.com/openPMD/openPMD-api.git"
+    CACHE STRING
+    "Repository URI to pull and build openPMD-api from if(ImpactX_openpmd_internal)")
+set(ImpactX_openpmd_branch "7813b6f26c37216aaedda300cfd1c201f7adad25"
+    CACHE STRING
+    "Repository branch for ImpactX_openPMD_repo if(ImpactX_openpmd_internal)")
+
 find_ablastr()

docs/Doxyfile

@@ -2335,7 +2335,8 @@ SKIP_FUNCTION_MACROS   = YES
 # run, you must also specify the path to the tagfile here.
 
 TAGFILES               = amrex-doxygen-web.tag.xml=https://amrex-codes.github.io/amrex/doxygen/  \
-                         warpx-doxygen-web.tag.xml=https://warpx.readthedocs.io/en/latest/_static/doxyhtml/
+                         warpx-doxygen-web.tag.xml=https://warpx.readthedocs.io/en/latest/_static/doxyhtml/  \
+                         openpmd-api-doxygen-web.tag.xml=https://openpmd-api.readthedocs.io/en/latest/_static/doxyhtml/
 
 # When a file name is specified after GENERATE_TAGFILE, doxygen will create a
 # tag file that is based on the input files it reads. See section "Linking to

docs/source/conf.py

@@ -190,7 +190,7 @@
 # Tell sphinx what the pygments highlight language should be.
 highlight_language = "cpp"
 
-# Download AMReX & WarpX Doxygen Tagfile to interlink Doxygen docs
+# Download AMReX & WarpX & openPMD-api Doxygen Tagfiles to interlink Doxygen docs
 url = "https://amrex-codes.github.io/amrex/docs_xml/doxygen/amrex-doxygen-web.tag.xml"
 urllib.request.urlretrieve(url, "../amrex-doxygen-web.tag.xml")
 
@@ -199,6 +199,9 @@
 )
 urllib.request.urlretrieve(url, "../warpx-doxygen-web.tag.xml")
 
+url = "https://openpmd-api.readthedocs.io/en/latest/_static/doxyhtml/openpmd-api-doxygen-web.tag.xml"
+urllib.request.urlretrieve(url, "../openpmd-api-doxygen-web.tag.xml")
+
 # Build Doxygen
 subprocess.call(
     "cd ../; doxygen;"

docs/source/install/cmake.rst

@@ -107,6 +107,7 @@ CMake Option                    Default & Values                             Des
 ``ImpactX_LIB``                 ON/**OFF**                                   Build ImpactX as a library (shared or static)
 ``ImpactX_MPI``                 **ON**/OFF                                   Multi-node support (message-passing)
 ``ImpactX_MPI_THREAD_MULTIPLE`` **ON**/OFF                                   MPI thread-multiple support, i.e. for ``async_io``
+``ImpactX_OPENPMD``             **ON**/OFF                                   openPMD I/O (HDF5, ADIOS)
 ``ImpactX_PRECISION``           SINGLE/**DOUBLE**                            Floating point precision (single/double)
 ``ImpactX_PYTHON``              ON/**OFF**                                   Python bindings
 ``Python_EXECUTABLE``           (newest found)                               Path to Python executable
@@ -120,6 +121,7 @@ By default, the most important dependencies of ImpactX are automatically downloa
 ============================= ============================================== ===========================================================
 CMake Option                  Default & Values                               Description
 ============================= ============================================== ===========================================================
+``BUILD_SHARED_LIBS``         ON/**OFF**                                     Build shared libraries for dependencies
 ``CCACHE_PROGRAM``            First found ``ccache`` executable.             Set to ``-DCCACHE_PROGRAM=NO`` to disable CCache.
 ``ImpactX_ablastr_src``       *None*                                         Path to ABLASTR source directory (preferred if set)
 ``ImpactX_ablastr_repo``      ``https://github.com/ECP-WarpX/WarpX.git``     Repository URI to pull and build ABLASTR from
@@ -129,6 +131,10 @@ CMake Option                  Default & Values                               Des
 ``ImpactX_amrex_repo``        ``https://github.com/AMReX-Codes/amrex.git``   Repository URI to pull and build AMReX from
 ``ImpactX_amrex_branch``      *we set and maintain a compatible commit*      Repository branch for ``ImpactX_amrex_repo``
 ``ImpactX_amrex_internal``    **ON**/OFF                                     Needs a pre-installed AMReX library if set to ``OFF``
+``ImpactX_openpmd_src``       *None*                                         Path to openPMD-api source directory (preferred if set)
+``ImpactX_openpmd_repo``      ``https://github.com/openPMD/openPMD-api.git`` Repository URI to pull and build openPMD-api from
+``ImpactX_openpmd_branch``    *we set and maintain a compatible commit*                                     Repository branch for ``ImpactX_openpmd_repo``
+``ImpactX_openpmd_internal``  **ON**/OFF                                     Needs a pre-installed openPMD-api library if set to ``OFF``
 ``ImpactX_pyamrex_src``       *None*                                         Path to AMReX source directory (preferred if set)
 ``ImpactX_pyamrex_repo``      ``https://github.com/AMReX-Codes/pyamrex.git`` Repository URI to pull and build pyAMReX from
 ``ImpactX_pyamrex_branch``    *we set and maintain a compatible commit*      Repository branch for ``ImpactX_pyamrex_repo``

docs/source/install/dependencies.rst

@@ -18,7 +18,7 @@ Optional dependencies include:
 - `CUDA Toolkit 11.0+ <https://developer.nvidia.com/cuda-downloads>`__: for Nvidia GPU support (see `matching host-compilers <https://gist.github.com/ax3l/9489132>`_)
 - `OpenMP 3.1+ <https://www.openmp.org>`__: for threaded CPU execution
 - `FFTW3 <http://www.fftw.org>`_: for spectral solver support
-- `openPMD-api 0.14.2+ <https://github.com/openPMD/openPMD-api>`__: we automatically download and compile a copy of openPMD-api for openPMD I/O support
+- `openPMD-api 0.15.0+ <https://github.com/openPMD/openPMD-api>`__: we automatically download and compile a copy of openPMD-api for openPMD I/O support
 
   - see `optional I/O backends <https://github.com/openPMD/openPMD-api#dependencies>`__
 - `CCache <https://ccache.dev>`__: to speed up rebuilds (needs 3.7.9+ for CUDA)