Skip to content

Commit 3278fe9

Browse files
cxp484cxp484
authored
Merge pull request #14564 from cxp484/FireX
FireX FDS Build: Adding FRONTIER ORNL scripts
2 parents 9263a00 + eed875f commit 3278fe9

File tree

5 files changed

+132
-2
lines changed

5 files changed

+132
-2
lines changed

‎Verification/GPU_Tests/FRONTIER_ORNL_runjob.sh‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
#!/bin/bash
2+
3+
#!/bin/bash
4+
#SBATCH -A ENG146
5+
#SBATCH -J test_gpu
6+
#SBATCH -o %x-%j.out
7+
#SBATCH -e %x-%j.err
8+
#SBATCH -t 1:00:00
9+
#SBATCH -q debug
10+
#SBATCH -N 1
11+
12+
module load PrgEnv-gnu
13+
module load rocm
14+
module load craype-accel-amd-gfx90a
15+
export MPICH_GPU_SUPPORT_ENABLED=1
16+
export FDS_RANKS_PER_GPU=8
17+
18+
19+
20+
FDSEXEC=~/firemodels/firex/fds/Build/ompi_gnu_linux/fds_ompi_gnu_linux
21+
echo "Launching application..."
22+
echo " Input file: test_gpu.fds"
23+
echo " Exec file: ${FDSEXEC}"
24+
echo "FDS_RANKS_PER_GPU: ${FDS_RANKS_PER_GPU}"
25+
echo " Directory: `pwd`"
26+
echo " Host: `hostname`"
27+
echo "started running at `date`" >> test_gpu.qlog
28+
cd /ccs/home/isapchp/firemodels/firex/fds/Verification/GPU_Tests
29+
#srun -N 1 -n 8 -c 1 -m block:block --ntasks-per-gpu=8 --gpus-per-node=1 --gpu-bind=none ${FDSEXEC} test_gpu.fds | sort
30+
srun -n 8 -c 1 -m block:block --gpus-per-task=1 --gpu-bind=closest ${FDSEXEC} test_gpu.fds | sort
31+
echo "finished running at `date`" >> test_gpu.qlog
32+

‎Verification/GPU_Tests/HYPRE_GPU_SCALING/FRONTIER_ORNL_runjob.sh‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
#!/bin/bash
2+
3+
#!/bin/bash
4+
#SBATCH -A ENG146
5+
#SBATCH -J test_gpu
6+
#SBATCH -o %x-%j.out
7+
#SBATCH -e %x-%j.err
8+
#SBATCH -t 2:00:00
9+
#SBATCH -p batch
10+
#SBATCH -N 1
11+
12+
module load PrgEnv-gnu
13+
module load rocm
14+
module load craype-accel-amd-gfx90a
15+
export MPICH_GPU_SUPPORT_ENABLED=1
16+
export FDS_RANKS_PER_GPU=8
17+
18+
19+
# MPI and OpenMP settings
20+
NNODES=1
21+
NRANKS_PER_NODE=${NUM_OF_MESHES} # Meshes
22+
NTHREADS=1
23+
24+
NTOTRANKS=$(( NNODES * NRANKS_PER_NODE ))
25+
echo "NUM_OF_NODES= ${NNODES} TOTAL_NUM_RANKS= ${NTOTRANKS} RANKS_PER_NODE= ${NRANKS_PER_NODE} THREADS_PER_RANK= ${NTHREADS}"
26+
27+
export FDS_RANKS_PER_GPU=${RANKS_PER_GPU}
28+
29+
if [ "$RANKS_PER_GPU" -eq 1 ]; then
30+
CPARAM=1
31+
else
32+
CPARAM=$(( 8 / RANKS_PER_GPU ))
33+
if [ "$CPARAM" -lt 1 ]; then
34+
CPARAM=1
35+
elif [ "$CPARAM" -gt 7 ]; then
36+
CPARAM=7
37+
fi
38+
fi
39+
40+
FDSEXEC=~/firemodels/firex/fds/Build/ompi_gnu_linux/fds_ompi_gnu_linux
41+
FDS_BASENAME="${FDS_CASE_FILE%.fds}" # removes .fds suffix
42+
echo "Launching application..."
43+
echo " Input file: ${FDS_CASE_FILE}"
44+
echo " NUM_OF_MESHES: ${NUM_OF_MESHES}"
45+
echo " Exec file: ${FDSEXEC}"
46+
echo "FDS_RANKS_PER_GPU: ${FDS_RANKS_PER_GPU}"
47+
echo " CPARAM: ${CPARAM}"
48+
echo " Directory: `pwd`"
49+
echo " Host: `hostname`"
50+
echo "started running at `date`" >> ${FDS_BASENAME}.qlog
51+
cd /ccs/home/isapchp/firemodels/firex/fds/Verification/GPU_Tests/HYPRE_GPU_SCALING
52+
#srun -n ${NTOTRANKS} -c 1 --ntasks-per-gpu=${FDS_RANKS_PER_GPU} --gpu-bind=closest ${FDSEXEC} ${FDS_CASE_FILE}
53+
srun -n ${NTOTRANKS} -c ${CPARAM} -m block:block --gpus-per-task=1 --gpu-bind=closest ${FDSEXEC} ${FDS_CASE_FILE}
54+
echo "finished running at `date`" >> ${FDS_BASENAME}.qlog
55+

‎Verification/GPU_Tests/HYPRE_GPU_SCALING/POLARIS_ALCF_runjob.sh‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ NTOTRANKS=$(( NNODES * NRANKS_PER_NODE ))
2626
echo "NUM_OF_NODES= ${NNODES} TOTAL_NUM_RANKS= ${NTOTRANKS} RANKS_PER_NODE= ${NRANKS_PER_NODE} THREADS_PER_RANK= ${NTHREADS}"
2727

2828
FDSEXEC=~/firemodels/firex/fds/Build/ompi_gnu_linux/fds_ompi_gnu_linux
29+
FDS_BASENAME="${FDS_CASE_FILE%.fds}" # removes .fds suffix
2930
echo "Launching application..."
3031
echo " Input file: ${FDS_CASE_FILE}"
3132
echo " NUM_OF_MESHES: ${NUM_OF_MESHES}"
@@ -38,4 +39,4 @@ echo "started running at `date`" >> ${FDS_BASENAME}.qlog
3839
# For applications that internally handle binding MPI/OpenMP processes to GPUs
3940
cd /home/isapchp/firemodels/firex/fds/Verification/GPU_Tests/HYPRE_GPU_SCALING
4041
mpiexec -n ${NTOTRANKS} --ppn ${NRANKS_PER_NODE} --depth=${NDEPTH} --cpu-bind depth --env OMP_NUM_THREADS=${NTHREADS} -env OMP_PLACES=threads ./set_affinity_gpu_polaris.sh ${FDSEXEC} ${FDS_CASE_FILE}
41-
42+
echo "finished running at `date`" >> ${FDS_BASENAME}.qlog

‎Verification/GPU_Tests/HYPRE_GPU_SCALING/submit_all.sh‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,48 @@ elif [ "$HPCSYS" == "POLARIS" ]; then
4646
qsub -v NUM_OF_MESHES=8,RANKS_PER_GPU=1,FDS_CASE_FILE=test_8mesh_NOFRPG.fds POLARIS_ALCF_runjob.sh
4747
qsub -v NUM_OF_MESHES=16,RANKS_PER_GPU=1,FDS_CASE_FILE=test_16mesh_NOFRPG.fds POLARIS_ALCF_runjob.sh
4848
qsub -v NUM_OF_MESHES=32,RANKS_PER_GPU=1,FDS_CASE_FILE=test_32mesh_NOFRPG.fds POLARIS_ALCF_runjob.sh
49+
50+
elif [ "$HPCSYS" == "FRONTIER" ]; then
51+
export NUM_OF_MESHES=4
52+
export RANKS_PER_GPU=4
53+
export FDS_CASE_FILE=test_4mesh_RS4.fds
54+
sbatch FRONTIER_ORNL_runjob.sh
55+
56+
export NUM_OF_MESHES=8
57+
export RANKS_PER_GPU=2
58+
export FDS_CASE_FILE=test_8mesh_RS2.fds
59+
sbatch FRONTIER_ORNL_runjob.sh
60+
61+
export NUM_OF_MESHES=16
62+
export RANKS_PER_GPU=4
63+
export FDS_CASE_FILE=test_16mesh_RS4.fds
64+
sbatch FRONTIER_ORNL_runjob.sh
65+
66+
export NUM_OF_MESHES=32
67+
export RANKS_PER_GPU=8
68+
export FDS_CASE_FILE=test_32mesh_RS8.fds
69+
sbatch FRONTIER_ORNL_runjob.sh
70+
71+
export NUM_OF_MESHES=4
72+
export RANKS_PER_GPU=1
73+
export FDS_CASE_FILE=test_4mesh.fds
74+
sbatch FRONTIER_ORNL_runjob.sh
75+
76+
export NUM_OF_MESHES=8
77+
export RANKS_PER_GPU=1
78+
export FDS_CASE_FILE=test_8mesh_NOFRPG.fds
79+
sbatch FRONTIER_ORNL_runjob.sh
80+
81+
export NUM_OF_MESHES=16
82+
export RANKS_PER_GPU=1
83+
export FDS_CASE_FILE=test_16mesh_NOFRPG.fds
84+
sbatch FRONTIER_ORNL_runjob.sh
85+
86+
export NUM_OF_MESHES=32
87+
export RANKS_PER_GPU=1
88+
export FDS_CASE_FILE=test_32mesh_NOFRPG.fds
89+
sbatch FRONTIER_ORNL_runjob.sh
90+
4991
fi
5092

5193

‎Verification/GPU_Tests/VISTA_TACC_runjob.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ sleep 5 # Wait for daemons to initialize
2929
# 2. Run your CUDA application
3030
export UCX_RCACHE_ENABLE=n
3131
#export UCX_LOG_LEVEL=info
32-
export OMP_NUM_THREADS=8
32+
export OMP_NUM_THREADS=1
3333
export FDS_RANKS_PER_GPU=8
3434
FDSEXEC=~/firemodels/firex/fds/Build/ompi_gnu_linux/fds_ompi_gnu_linux
3535

0 commit comments

Comments
 (0)