Add profiling section for LUMI

Added profiling instructions for LUMI using rocprof and omniperf.

Author: aaadelmann

README.md

@@ -318,6 +318,86 @@ srun ./select_gpu ${EXE_DIR}/TestGaussian 1024 1024 1024 pencils a2av no-reorder
 rm -rf ./select_gpu
 ```
 
+# Profiling on LUMI
+
+## rocprof
+
+```
+#!/bin/bash -l
+#
+#SBATCH --job-name=opalx1
+#SBATCH --error=opalx-%j.error
+#SBATCH --output=opalx-2-%j.out
+#SBATCH --time=00:05:00
+#SBATCH --partition=standard-g
+#SBATCH --nodes 1
+#SBATCH --ntasks-per-core=1
+#SBATCH -c 56 --threads-per-core=1
+#SBATCH --ntasks-per-node=1
+#SBATCH --gpus-per-node=8
+#SBATCH --account=project_465001705 
+#SBATCH --hint=nomultithread
+#SBATCH --hint=exclusive
+CPU_BIND="map_cpu:49,57,17,25,1,9,33,41"
+export MPICH_GPU_SUPPORT_ENABLED=1
+ 
+ulimit -s unlimited
+export EXE_DIR=/users/adelmann/sandbox/opalx/build/src/
+module load cray-python/3.11.7 
+module use /appl/local/containers/test-modules
+module load LUMI/24.03 partition/G cpeAMD rocm/6.1.3 buildtools/24.03
+
+cat << EOF > select_gpu
+#!/bin/bash
+export HIP_VISIBLE_DEVICES=\$SLURM_LOCALID
+exec \$*
+EOF
+chmod +x ./select_gpu
+srun ./select_gpu rocprof --hip-trace ${EXE_DIR}/opalx input.in --info 5
+rm -rf ./select_gpu
+
+```
+
+
+## omniperf
+
+```
+#!/bin/bash -l
+#
+#SBATCH --job-name=opalx1
+#SBATCH --error=opalx-%j.error
+#SBATCH --output=opalx-2-%j.out
+#SBATCH --time=00:05:00
+#SBATCH --partition=standard-g
+#SBATCH --nodes 1
+#SBATCH --ntasks-per-core=1
+#SBATCH -c 56 --threads-per-core=1
+#SBATCH --ntasks-per-node=1
+#SBATCH --gpus-per-node=8
+#SBATCH --account=project_465001705 
+#SBATCH --hint=nomultithread
+#SBATCH --hint=exclusive
+CPU_BIND="map_cpu:49,57,17,25,1,9,33,41"
+export MPICH_GPU_SUPPORT_ENABLED=1
+ 
+ulimit -s unlimited
+export EXE_DIR=/users/adelmann/sandbox/opalx/build/src/
+module load cray-python/3.11.7 
+module use /appl/local/containers/test-modules
+module load LUMI/24.03 partition/G cpeAMD rocm/6.1.3 buildtools/24.03
+module load omniperf
+cat << EOF > select_gpu
+#!/bin/bash
+#export ROCR_VISIBLE_DEVICES=\$SLURM_LOCALID
+export HIP_VISIBLE_DEVICES=\$SLURM_LOCALID
+exec \$*
+EOF
+chmod +x ./select_gpu
+srun ./select_gpu omniperf profile --name opalx  --roof-only --kernel-names -- ${EXE_DIR}/opalx input.in --info 5
+rm -rf ./select_gpu
+```
+
+
 # Profiling IPPL MPI calls
 
 You can use the mpiP tool (https://github.com/LLNL/mpiP) to get statistics about the MPI calls in IPPL.