diff --git a/Docs/source/install/hpc/lassen.rst b/Docs/source/install/hpc/lassen.rst index 4d30b2a75eb..41828d47b50 100644 --- a/Docs/source/install/hpc/lassen.rst +++ b/Docs/source/install/hpc/lassen.rst @@ -40,23 +40,23 @@ Use the following commands to download the WarpX source code: git clone https://github.com/BLAST-WarpX/warpx.git /usr/workspace/${USER}/lassen/src/warpx -We use system software modules, add environment hints and further dependencies via the file ``$HOME/lassen_v100_warpx_toss3.profile``. +We use system software modules, add environment hints and further dependencies via the file ``$HOME/lassen_v100_warpx.profile``. Create it now: .. code-block:: bash - cp /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example $HOME/lassen_v100_warpx_toss3.profile + cp /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example $HOME/lassen_v100_warpx.profile .. dropdown:: Script Details :color: light :icon: info :animate: fade-in-slide-down - .. literalinclude:: ../../../../Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example + .. literalinclude:: ../../../../Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example :language: bash Edit the 2nd line of this script, which sets the ``export proj=""`` variable. -For example, if you are member of the project ``nsldt``, then run ``vi $HOME/lassen_v100_warpx_toss3.profile``. +For example, if you are member of the project ``nsldt``, then run ``vi $HOME/lassen_v100_warpx.profile``. Enter the edit mode by typing ``i`` and edit line 2 to read: .. code-block:: bash @@ -71,21 +71,21 @@ Exit the ``vi`` editor with ``Esc`` and then type ``:wq`` (write & quit). .. code-block:: bash - source $HOME/lassen_v100_warpx_toss3.profile + source $HOME/lassen_v100_warpx.profile Finally, since lassen does not yet provide software modules for some of our dependencies, install them once: .. code-block:: bash - bash /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh - source /usr/workspace/${USER}/lassen-toss3/gpu/venvs/warpx-lassen-toss3/bin/activate + bash /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/install_v100_dependencies.sh + source /usr/workspace/${USER}/lassen/gpu/venvs/warpx-lassen/bin/activate .. dropdown:: Script Details :color: light :icon: info :animate: fade-in-slide-down - .. literalinclude:: ../../../../Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh + .. literalinclude:: ../../../../Tools/machines/lassen-llnl/install_v100_dependencies.sh :language: bash @@ -141,7 +141,7 @@ If you already installed WarpX in the past and want to update it, start by getti git status git log # press q to exit -And, if needed, +Then - :ref:`update the lassen_v100_warpx.profile file `, - log out and into the system, activate the now updated environment profile as usual, diff --git a/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh b/Tools/machines/lassen-llnl/install_v100_dependencies.sh similarity index 86% rename from Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh rename to Tools/machines/lassen-llnl/install_v100_dependencies.sh index 32106638721..bf5c79c559d 100644 --- a/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh +++ b/Tools/machines/lassen-llnl/install_v100_dependencies.sh @@ -15,13 +15,13 @@ set -eu -o pipefail # Check: ###################################################################### # # Was lassen_v100_warpx.profile sourced and configured correctly? -if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your lassen_v100_warpx_toss3.profile file! Please edit its line 2 to continue!"; exit 1; fi +if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your lassen_v100_warpx.profile file! Please edit its line 2 to continue!"; exit 1; fi # Remove old dependencies ##################################################### # -SRC_DIR="/usr/workspace/${USER}/lassen-toss3/src" -SW_DIR="/usr/workspace/${USER}/lassen-toss3/gpu" +SRC_DIR="/usr/workspace/${USER}/lassen/src" +SW_DIR="/usr/workspace/${USER}/lassen/gpu" rm -rf ${SW_DIR} mkdir -p ${SW_DIR} mkdir -p ${SRC_DIR} @@ -110,22 +110,24 @@ cmake --build ${build_dir}/lapackpp-lassen-build --target install --parallel 10 export PIP_EXTRA_INDEX_URL="https://pypi.org/simple" python3 -m pip install --upgrade --user virtualenv -rm -rf ${SW_DIR}/venvs/warpx-lassen-toss3 -python3 -m venv ${SW_DIR}/venvs/warpx-lassen-toss3 -source ${SW_DIR}/venvs/warpx-lassen-toss3/bin/activate +rm -rf ${SW_DIR}/venvs/warpx-lassen +python3 -m venv ${SW_DIR}/venvs/warpx-lassen +source ${SW_DIR}/venvs/warpx-lassen/bin/activate python3 -m pip install --upgrade pip # python3 -m pip cache purge # error: pip cache commands can not function since cache is disabled python3 -m pip install --upgrade build python3 -m pip install --upgrade packaging python3 -m pip install --upgrade wheel python3 -m pip install --upgrade setuptools[core] -python3 -m pip install --upgrade cython +python3 -m pip install --upgrade scikit-build-core +python3 -m pip install --upgrade "cython>=3" python3 -m pip install --upgrade numpy python3 -m pip install --upgrade pandas CMAKE_PREFIX_PATH=/usr/lib64:${CMAKE_PREFIX_PATH} python3 -m pip install --upgrade -Ccompile-args="-j10" -Csetup-args=-Dblas=BLAS -Csetup-args=-Dlapack=BLAS scipy -python3 -m pip install --upgrade mpi4py --no-cache-dir --no-build-isolation --no-binary mpi4py +# Lassen does not support MPI v4+ yet +MPI4PY_BUILD_BACKEND="scikit-build-core" python3 -m pip install --upgrade "mpi4py<4" --no-cache-dir --no-build-isolation --no-binary mpi4py python3 -m pip install --upgrade openpmd-api -CC=mpicc H5PY_SETUP_REQUIRES=0 HDF5_DIR=${SW_DIR}/hdf5-1.14.1.2 HDF5_MPI=ON python3 -m pip install --upgrade h5py --no-cache-dir --no-build-isolation --no-binary h5py +CC=mpigcc H5PY_SETUP_REQUIRES=0 HDF5_DIR=${SW_DIR}/hdf5-1.14.1.2 HDF5_MPI=ON python3 -m pip install --upgrade h5py --no-cache-dir --no-build-isolation --no-binary h5py MPLLOCALFREETYPE=1 python3 -m pip install --upgrade matplotlib==3.2.2 # does not try to build freetype itself echo "matplotlib==3.2.2" > ${build_dir}/constraints.txt python3 -m pip install --upgrade -c ${build_dir}/constraints.txt yt diff --git a/Tools/machines/lassen-llnl/lassen_v100.bsub b/Tools/machines/lassen-llnl/lassen_v100.bsub index 1b094cc34f9..49c6701dab0 100644 --- a/Tools/machines/lassen-llnl/lassen_v100.bsub +++ b/Tools/machines/lassen-llnl/lassen_v100.bsub @@ -54,4 +54,4 @@ export OMP_NUM_THREADS=1 jsrun -r 4 -a1 -g 1 -c 7 -e prepended hostname > task_host_mapping.txt # run WarpX -jsrun -r 4 -a 1 -g 1 -c 7 -l GPU-CPU -d packed -b rs -e prepended -M "-gpu" > output.txt +jsrun -r 4 -a 1 -g 1 -c 7 -l GPU-CPU -d packed -b rs -e prepended > output.txt diff --git a/Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example b/Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example similarity index 92% rename from Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example rename to Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example index a2b9356104b..883d2caf73e 100644 --- a/Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example +++ b/Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example @@ -11,7 +11,7 @@ module load boost/1.70.0 # optional: for openPMD support SRC_DIR="/usr/workspace/${USER}/lassen/src" -SW_DIR="/usr/workspace/${USER}/lassen-toss3/gpu" +SW_DIR="/usr/workspace/${USER}/lassen/gpu" export CMAKE_PREFIX_PATH=${SW_DIR}/c-blosc-1.21.1:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${SW_DIR}/hdf5-1.14.1.2:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${SW_DIR}/adios2-2.10.2:$CMAKE_PREFIX_PATH @@ -30,9 +30,9 @@ export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH # optional: for Python bindings module load python/3.11.5 -if [ -d "${SW_DIR}/venvs/warpx-lassen-toss3" ] +if [ -d "${SW_DIR}/venvs/warpx-lassen" ] then - source ${SW_DIR}/venvs/warpx-lassen-toss3/bin/activate + source ${SW_DIR}/venvs/warpx-lassen/bin/activate fi # optional: an alias to request an interactive node for two hours