Merge pull request #13668 from marcosvanella/master

FDS Utilities : Misc/loop_test, setup make and run for gnu, ifort and…

Author: marcosvanella

Utilities/Misc/loop_test/README

@@ -0,0 +1,10 @@
+The code in main.f90 runs 640 intances of the 3D loops for computation of 
+momentum RHS done in VELOCITY_FLUX. This is done on a single mesh with 64^3 cells.
+----------------------------------------------------------------------------------
+In Spark run the script:
+
+$./make_run_all.sh
+
+To check calculation times given by CPU_TIME() type:
+
+$grep Time *.txt

Utilities/Misc/loop_test/main.f90

@@ -5,7 +5,8 @@ PROGRAM LOOP3D
 
 ! Miscellaneous declarations
 INTEGER, PARAMETER :: EB = SELECTED_REAL_KIND(12)
-INTEGER, PARAMETER :: IBAR = 256, JBAR = 256, KBAR = 256
+INTEGER, PARAMETER :: IBAR = 64, JBAR = 64, KBAR = 64
+INTEGER, PARAMETER :: NSOLVES = 640
 INTEGER, PARAMETER :: NEDGE = 12
 INTEGER, PARAMETER :: IBP1 = IBAR+1, JBP1 = JBAR+1, KBP1 = KBAR+1
 REAL(EB), PARAMETER :: FOTH = 4.0_EB/3.0_EB
@@ -20,7 +21,7 @@ PROGRAM LOOP3D
             DUDX,DVDY,DWDZ,DUDY,DUDZ,DVDX,DVDZ,DWDX,DWDY, &
             VOMZ,WOMY,UOMY,VOMX,UOMZ,WOMX, &
             RRHO,TXXP,TXXM,TYYP,TYYM,TZZP,TZZM,DTXXDX,DTYYDY,DTZZDZ,T_NOW,T_END
-INTEGER :: I,J,K,IEXP,IEXM,IEYP,IEYM,IEZP,IEZM,IC,IC1,IC2,IE,MAX_EDGE,NT
+INTEGER :: I,J,K,IEXP,IEXM,IEYP,IEYM,IEZP,IEZM,IC,IC1,IC2,IE,MAX_EDGE,NT,ISOLVE
 CHARACTER(LEN=50) :: FILENAME
 
 TYPE CELL_TYPE
@@ -34,6 +35,7 @@ PROGRAM LOOP3D
 END TYPE EDGE_TYPE
 TYPE(EDGE_TYPE), ALLOCATABLE, DIMENSION(:) :: EDGE
 
+NT=1
 ! Write out Starting:
 !$OMP PARALLEL
 !$OMP MASTER
@@ -42,7 +44,14 @@ PROGRAM LOOP3D
 !$OMP BARRIER
 !$OMP END PARALLEL
 
-WRITE(FILENAME,'(A,I4.4,A,I2.2,A)') 'loop3d_',IBAR,'_',NT,'THR.txt'
+WRITE(FILENAME,'(A,I4.4,A,I2.2,A)') 'loop3d_gnu_',IBAR,'_',NT,'THR.txt'
+#ifdef IFORT
+WRITE(FILENAME,'(A,I4.4,A,I2.2,A)') 'loop3d_ifort_',IBAR,'_',NT,'THR.txt'
+#endif
+#ifdef IFX
+WRITE(FILENAME,'(A,I4.4,A,I2.2,A)') 'loop3d_ifx_',IBAR,'_',NT,'THR.txt'
+#endif
+
 WRITE(*,*) 'Starting Loop3D, out file: ',TRIM(FILENAME)
 OPEN(UNIT=10,FILE=TRIM(FILENAME),STATUS='UNKNOWN')
 WRITE(10,*) 'Starting Loop3D'
@@ -175,7 +184,9 @@ PROGRAM LOOP3D
 ENDDO
 
 CALL CPU_TIME(T_NOW)
-CALL LOOP3D_OMP_CPU()
+DO ISOLVE = 1, NSOLVES
+   CALL LOOP3D_OMP_CPU()
+ENDDO
 CALL CPU_TIME(T_END)
 
 WRITE(10,*) 'Time=',T_END-T_NOW

Utilities/Misc/loop_test/make_gnu.sh

@@ -1 +1 @@
-gfortran -m64 -O2 -fopenmp main.f90 -o loop3d
+gfortran -m64 -O2 -fopenmp -cpp main.f90 -o loop3d

Utilities/Misc/loop_test/make_run_all.sh

@@ -0,0 +1,25 @@
+#!/bin/bash 
+# For use in spark.el.nist.gov:
+rm -f loop3d_gnu loop3d_ifort loop3d_ifx *.txt
+# GNU - gfortran:
+module load gcc/13.1.1
+echo "Compiling loop3d_gnu.."
+gfortran -m64 -O3 -cpp  main.f90 -o loop3d_gnu
+echo "Submitting loop3d_gnu job.."
+sbatch submit_gnu.slog
+
+#Intel:
+module unload gcc/13.1.1
+module load tbb/latest compiler-rt/latest compiler/latest
+# ifort:
+echo "Compiling loop3d_ifort.."
+ifort -m64 -O2 -ipo -fpp -DIFORT -diag-disable=10448 main.f90 -o loop3d_ifort
+echo "Submitting loop3d_ifort job.."
+sbatch submit_ifort.slog
+
+# ifx:
+echo "Compiling loop3d_ifx.."
+ifx -m64 -O2 -ipo -fpp -DIFX main.f90 -o loop3d_ifx
+echo "Submitting loop3d_ifx job.."
+sbatch submit_ifx.slog
+

Utilities/Misc/loop_test/submit_gnu.slog

@@ -0,0 +1,13 @@
+#!/bin/bash
+#SBATCH -J loop3d_gnu
+#SBATCH -e /home/mnv/FireModels_fork_home/fds/Utilities/Misc/loop_test/loop3d_gnu.err
+#SBATCH -o /home/mnv/FireModels_fork_home/fds/Utilities/Misc/loop_test/loop3d_gnu.log
+#SBATCH --partition=firebot
+#SBATCH --ntasks=1
+#SBATCH --exclusive
+#SBATCH --cpus-per-task=1
+#SBATCH --time=99-99:99:99
+export OMP_NUM_THREADS=1
+
+cd /home/mnv/FireModels_fork_home/fds/Utilities/Misc/loop_test 
+./loop3d_gnu

Utilities/Misc/loop_test/submit_ifort.slog

@@ -0,0 +1,13 @@
+#!/bin/bash
+#SBATCH -J loop3d_ifort
+#SBATCH -e /home/mnv/FireModels_fork_home/fds/Utilities/Misc/loop_test/loop3d_ifort.err
+#SBATCH -o /home/mnv/FireModels_fork_home/fds/Utilities/Misc/loop_test/loop3d_ifort.log
+#SBATCH --partition=firebot
+#SBATCH --ntasks=1
+#SBATCH --exclusive
+#SBATCH --cpus-per-task=1
+#SBATCH --time=99-99:99:99
+export OMP_NUM_THREADS=1
+
+cd /home/mnv/FireModels_fork_home/fds/Utilities/Misc/loop_test 
+./loop3d_ifort

Utilities/Misc/loop_test/submit_ifx.slog

@@ -0,0 +1,13 @@
+#!/bin/bash
+#SBATCH -J loop3d_ifx
+#SBATCH -e /home/mnv/FireModels_fork_home/fds/Utilities/Misc/loop_test/loop3d_ifx.err
+#SBATCH -o /home/mnv/FireModels_fork_home/fds/Utilities/Misc/loop_test/loop3d_ifx.log
+#SBATCH --partition=firebot
+#SBATCH --ntasks=1
+#SBATCH --exclusive
+#SBATCH --cpus-per-task=1
+#SBATCH --time=99-99:99:99
+export OMP_NUM_THREADS=1
+
+cd /home/mnv/FireModels_fork_home/fds/Utilities/Misc/loop_test 
+./loop3d_ifx