diff --git a/Utilities/Misc/loop_test/README b/Utilities/Misc/loop_test/README new file mode 100644 index 00000000000..bfa7a3bbbb2 --- /dev/null +++ b/Utilities/Misc/loop_test/README @@ -0,0 +1,10 @@ +The code in main.f90 runs 640 intances of the 3D loops for computation of +momentum RHS done in VELOCITY_FLUX. This is done on a single mesh with 64^3 cells. +---------------------------------------------------------------------------------- +In Spark run the script: + +$./make_run_all.sh + +To check calculation times given by CPU_TIME() type: + +$grep Time *.txt diff --git a/Utilities/Misc/loop_test/main.f90 b/Utilities/Misc/loop_test/main.f90 index 2a817d39a5b..368cdeafb32 100644 --- a/Utilities/Misc/loop_test/main.f90 +++ b/Utilities/Misc/loop_test/main.f90 @@ -5,7 +5,8 @@ PROGRAM LOOP3D ! Miscellaneous declarations INTEGER, PARAMETER :: EB = SELECTED_REAL_KIND(12) -INTEGER, PARAMETER :: IBAR = 256, JBAR = 256, KBAR = 256 +INTEGER, PARAMETER :: IBAR = 64, JBAR = 64, KBAR = 64 +INTEGER, PARAMETER :: NSOLVES = 640 INTEGER, PARAMETER :: NEDGE = 12 INTEGER, PARAMETER :: IBP1 = IBAR+1, JBP1 = JBAR+1, KBP1 = KBAR+1 REAL(EB), PARAMETER :: FOTH = 4.0_EB/3.0_EB @@ -20,7 +21,7 @@ PROGRAM LOOP3D DUDX,DVDY,DWDZ,DUDY,DUDZ,DVDX,DVDZ,DWDX,DWDY, & VOMZ,WOMY,UOMY,VOMX,UOMZ,WOMX, & RRHO,TXXP,TXXM,TYYP,TYYM,TZZP,TZZM,DTXXDX,DTYYDY,DTZZDZ,T_NOW,T_END -INTEGER :: I,J,K,IEXP,IEXM,IEYP,IEYM,IEZP,IEZM,IC,IC1,IC2,IE,MAX_EDGE,NT +INTEGER :: I,J,K,IEXP,IEXM,IEYP,IEYM,IEZP,IEZM,IC,IC1,IC2,IE,MAX_EDGE,NT,ISOLVE CHARACTER(LEN=50) :: FILENAME TYPE CELL_TYPE @@ -34,6 +35,7 @@ PROGRAM LOOP3D END TYPE EDGE_TYPE TYPE(EDGE_TYPE), ALLOCATABLE, DIMENSION(:) :: EDGE +NT=1 ! Write out Starting: !$OMP PARALLEL !$OMP MASTER @@ -42,7 +44,14 @@ PROGRAM LOOP3D !$OMP BARRIER !$OMP END PARALLEL -WRITE(FILENAME,'(A,I4.4,A,I2.2,A)') 'loop3d_',IBAR,'_',NT,'THR.txt' +WRITE(FILENAME,'(A,I4.4,A,I2.2,A)') 'loop3d_gnu_',IBAR,'_',NT,'THR.txt' +#ifdef IFORT +WRITE(FILENAME,'(A,I4.4,A,I2.2,A)') 'loop3d_ifort_',IBAR,'_',NT,'THR.txt' +#endif +#ifdef IFX +WRITE(FILENAME,'(A,I4.4,A,I2.2,A)') 'loop3d_ifx_',IBAR,'_',NT,'THR.txt' +#endif + WRITE(*,*) 'Starting Loop3D, out file: ',TRIM(FILENAME) OPEN(UNIT=10,FILE=TRIM(FILENAME),STATUS='UNKNOWN') WRITE(10,*) 'Starting Loop3D' @@ -175,7 +184,9 @@ PROGRAM LOOP3D ENDDO CALL CPU_TIME(T_NOW) -CALL LOOP3D_OMP_CPU() +DO ISOLVE = 1, NSOLVES + CALL LOOP3D_OMP_CPU() +ENDDO CALL CPU_TIME(T_END) WRITE(10,*) 'Time=',T_END-T_NOW diff --git a/Utilities/Misc/loop_test/make_gnu.sh b/Utilities/Misc/loop_test/make_gnu.sh index dd0f81b012a..8fd14905964 100755 --- a/Utilities/Misc/loop_test/make_gnu.sh +++ b/Utilities/Misc/loop_test/make_gnu.sh @@ -1 +1 @@ -gfortran -m64 -O2 -fopenmp main.f90 -o loop3d +gfortran -m64 -O2 -fopenmp -cpp main.f90 -o loop3d diff --git a/Utilities/Misc/loop_test/make_run_all.sh b/Utilities/Misc/loop_test/make_run_all.sh new file mode 100755 index 00000000000..f1f56213f81 --- /dev/null +++ b/Utilities/Misc/loop_test/make_run_all.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# For use in spark.el.nist.gov: +rm -f loop3d_gnu loop3d_ifort loop3d_ifx *.txt +# GNU - gfortran: +module load gcc/13.1.1 +echo "Compiling loop3d_gnu.." +gfortran -m64 -O3 -cpp main.f90 -o loop3d_gnu +echo "Submitting loop3d_gnu job.." +sbatch submit_gnu.slog + +#Intel: +module unload gcc/13.1.1 +module load tbb/latest compiler-rt/latest compiler/latest +# ifort: +echo "Compiling loop3d_ifort.." +ifort -m64 -O2 -ipo -fpp -DIFORT -diag-disable=10448 main.f90 -o loop3d_ifort +echo "Submitting loop3d_ifort job.." +sbatch submit_ifort.slog + +# ifx: +echo "Compiling loop3d_ifx.." +ifx -m64 -O2 -ipo -fpp -DIFX main.f90 -o loop3d_ifx +echo "Submitting loop3d_ifx job.." +sbatch submit_ifx.slog + diff --git a/Utilities/Misc/loop_test/submit_gnu.slog b/Utilities/Misc/loop_test/submit_gnu.slog new file mode 100644 index 00000000000..3f99f7f4b52 --- /dev/null +++ b/Utilities/Misc/loop_test/submit_gnu.slog @@ -0,0 +1,13 @@ +#!/bin/bash +#SBATCH -J loop3d_gnu +#SBATCH -e /home/mnv/FireModels_fork_home/fds/Utilities/Misc/loop_test/loop3d_gnu.err +#SBATCH -o /home/mnv/FireModels_fork_home/fds/Utilities/Misc/loop_test/loop3d_gnu.log +#SBATCH --partition=firebot +#SBATCH --ntasks=1 +#SBATCH --exclusive +#SBATCH --cpus-per-task=1 +#SBATCH --time=99-99:99:99 +export OMP_NUM_THREADS=1 + +cd /home/mnv/FireModels_fork_home/fds/Utilities/Misc/loop_test +./loop3d_gnu diff --git a/Utilities/Misc/loop_test/submit_ifort.slog b/Utilities/Misc/loop_test/submit_ifort.slog new file mode 100644 index 00000000000..7fcae04cf24 --- /dev/null +++ b/Utilities/Misc/loop_test/submit_ifort.slog @@ -0,0 +1,13 @@ +#!/bin/bash +#SBATCH -J loop3d_ifort +#SBATCH -e /home/mnv/FireModels_fork_home/fds/Utilities/Misc/loop_test/loop3d_ifort.err +#SBATCH -o /home/mnv/FireModels_fork_home/fds/Utilities/Misc/loop_test/loop3d_ifort.log +#SBATCH --partition=firebot +#SBATCH --ntasks=1 +#SBATCH --exclusive +#SBATCH --cpus-per-task=1 +#SBATCH --time=99-99:99:99 +export OMP_NUM_THREADS=1 + +cd /home/mnv/FireModels_fork_home/fds/Utilities/Misc/loop_test +./loop3d_ifort diff --git a/Utilities/Misc/loop_test/submit_ifx.slog b/Utilities/Misc/loop_test/submit_ifx.slog new file mode 100644 index 00000000000..2ad0bd39d18 --- /dev/null +++ b/Utilities/Misc/loop_test/submit_ifx.slog @@ -0,0 +1,13 @@ +#!/bin/bash +#SBATCH -J loop3d_ifx +#SBATCH -e /home/mnv/FireModels_fork_home/fds/Utilities/Misc/loop_test/loop3d_ifx.err +#SBATCH -o /home/mnv/FireModels_fork_home/fds/Utilities/Misc/loop_test/loop3d_ifx.log +#SBATCH --partition=firebot +#SBATCH --ntasks=1 +#SBATCH --exclusive +#SBATCH --cpus-per-task=1 +#SBATCH --time=99-99:99:99 +export OMP_NUM_THREADS=1 + +cd /home/mnv/FireModels_fork_home/fds/Utilities/Misc/loop_test +./loop3d_ifx