Skip to content

Commit 3cd7025

Browse files
hmarbonahmarbona
authored
Merge pull request #276 from hmarbona/master
2 parents 557d1e6 + 7453fb4 commit 3cd7025

24 files changed

Lines changed: 2348 additions & 670 deletions

File tree

‎.github/workflows/CI_parallel.yml‎

Lines changed: 158 additions & 138 deletions
Large diffs are not rendered by default.

‎Solver/configure‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ TEST_CASES="./Euler/BoxAroundCircle \
4444
./NavierStokes/Convergence_energy \
4545
./NavierStokes/Convergence_entropy \
4646
./NavierStokes/Cylinder \
47+
./NavierStokes/Cylinder_MLRK \
4748
./NavierStokes/CylinderFAS \
4849
./NavierStokes/CylinderAnisFAS \
4950
./NavierStokes/CylinderBR2 \

‎Solver/src/CahnHilliardSolver/main.f90‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -168,8 +168,8 @@ PROGRAM HORSES3DMainCH
168168
call Stopwatch % destruct
169169
CALL timeIntegrator % destruct()
170170
CALL sem % destruct()
171-
call Finalize_InterpolationMatrices
172171
call DestructBoundaryConditions
172+
call Finalize_InterpolationMatrices
173173
call DestructGlobalNodalStorage()
174174
CALL destructSharedBCModule
175175

‎Solver/src/MultiphaseSolver/SpatialDiscretization.f90‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -238,7 +238,6 @@ SUBROUTINE ComputeTimeDerivative( mesh, particles, time, mode, HO_Elements, elem
238238
!
239239
INTEGER :: k, eID, fID, i, j, ierr, locLevel, lID
240240
real(kind=RP) :: sqrtRho, invMa2
241-
class(Element), pointer :: e
242241
logical :: compute_element
243242
logical, allocatable :: face_mask(:)
244243
real(kind=RP) :: mu_smag, delta
@@ -280,7 +279,6 @@ SUBROUTINE ComputeTimeDerivative( mesh, particles, time, mode, HO_Elements, elem
280279
end do
281280
!$omp end do
282281
end select
283-
284282
!
285283
! -------------------------------
286284
! Set memory to Cahn-Hilliard (C)
@@ -550,7 +548,7 @@ SUBROUTINE ComputeTimeDerivative( mesh, particles, time, mode, HO_Elements, elem
550548
! Add the Non-Conservative term to QDot
551549
! -------------------------------------
552550
!
553-
!$omp do schedule(runtime) private(i,j,k,e,sqrtRho,invMa2,eID)
551+
!$omp do schedule(runtime) private(i,j,k,sqrtRho,invMa2,eID)
554552
do lID = 1, MLIter(locLevel,1) !
555553
eID = MLIter_eID(lID)
556554
compute_element = .true.

‎Solver/src/MultiphaseSolver/main.f90‎

Lines changed: 42 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,9 @@ PROGRAM HORSES3DMainMU
2828
TYPE( DGSem ) :: sem
2929
TYPE( TimeIntegrator_t ) :: timeIntegrator
3030
LOGICAL :: success, saveGradients, saveSource
31+
logical :: generateMonitor = .TRUE.
32+
logical :: optimizePartitionLevel=.FALSE.
33+
logical :: isMLRK=.FALSE.
3134
integer :: initial_iteration
3235
INTEGER :: ierr
3336
real(kind=RP) :: initial_time
@@ -88,23 +91,53 @@ PROGRAM HORSES3DMainMU
8891
call GetMeshPolynomialOrders(controlVariables,Nx,Ny,Nz,Nmax)
8992
call InitializeNodalStorage(controlVariables, Nmax)
9093
call Initialize_InterpolationMatrices(Nmax)
91-
94+
95+
! MPI Partition option for MLRK time integeration
96+
if ((MPI_Process % doMPIAction).and.(trim(controlVariables % stringValueForKey('explicit method', requestedLength = LINE_LENGTH)) == 'multi level rk3')) then
97+
98+
isMLRK = .TRUE. ! The time integration is Multi-Level RK
99+
100+
if ( controlVariables % ContainsKey("optimized partition") ) then
101+
optimizePartitionLevel = controlVariables % LogicalValueForKey ("optimized partition")
102+
end if
103+
104+
if (optimizePartitionLevel) generateMonitor =.FALSE. ! Do not generate monitor for the first construction of sem as it will be reconstruct
105+
end if
106+
107+
! Construct DGSEM library
92108
call sem % construct ( controlVariables = controlVariables, &
93-
Nx_ = Nx, Ny_ = Ny, Nz_ = Nz, &
94-
success = success)
95-
96-
call Initialize_SpaceAndTimeMethods(controlVariables, sem % mesh)
97-
98-
IF(.NOT. success) error stop "Mesh reading error"
99-
IF(.NOT. success) error stop "Boundary condition specification error"
100-
CALL UserDefinedFinalSetup(sem % mesh, thermodynamics, dimensionless, refValues, multiphase)
109+
Nx_ = Nx, Ny_ = Ny, Nz_ = Nz, &
110+
success = success, generateMonitor =generateMonitor)
101111
!
102112
! -------------------------
103113
! Set the initial condition
104114
! -------------------------
105115
!
116+
call UserDefinedFinalSetup(sem % mesh, thermodynamics, dimensionless, refValues, multiphase)
106117
call sem % SetInitialCondition(controlVariables, initial_iteration, initial_time)
107118
!
119+
! ----------------------------------------------
120+
! Reconstruct for MLRK explicit time step method
121+
! ----------------------------------------------
122+
!
123+
if (isMLRK .and. optimizePartitionLevel .and. MPI_Process % doMPIAction) then
124+
call sem % reconstruct ( controlVariables = controlVariables, &
125+
Nx_ = Nx, Ny_ = Ny, Nz_ = Nz, &
126+
success = success)
127+
call UserDefinedFinalSetup(sem % mesh, thermodynamics, dimensionless, refValues, multiphase)
128+
call sem % SetInitialCondition(controlVariables, initial_iteration, initial_time)
129+
end if
130+
!
131+
! -----------------------------
132+
! Initialize the discretization
133+
! -----------------------------
134+
!
135+
call Initialize_SpaceAndTimeMethods(controlVariables, sem % mesh)
136+
137+
IF(.NOT. success) error stop "Mesh reading error"
138+
IF(.NOT. success) error stop "Boundary condition specification error"
139+
140+
!
108141
! -----------------------------
109142
! Construct the time integrator
110143
! -----------------------------

‎Solver/src/NavierStokesSolver/main.f90‎

Lines changed: 43 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ PROGRAM HORSES3DMainNS
2020
use InterpolationMatrices , only: Initialize_InterpolationMatrices, Finalize_InterpolationMatrices
2121
use ProblemFileFunctions
2222
use BoundaryConditions , only: DestructBoundaryConditions
23+
use PartitionedMeshClass
2324
#ifdef _HAS_MPI_
2425
use mpi
2526
#endif
@@ -34,13 +35,17 @@ PROGRAM HORSES3DMainNS
3435
TYPE( DGSem ) :: sem
3536
TYPE( TimeIntegrator_t ) :: timeIntegrator
3637
LOGICAL :: success, saveGradients, saveSensor, saveLES, saveSource
38+
logical :: generateMonitor = .TRUE.
39+
logical :: optimizePartitionLevel=.FALSE.
40+
logical :: isMLRK=.FALSE.
3741
integer :: initial_iteration
3842
INTEGER :: ierr
3943
real(kind=RP) :: initial_time
4044
character(len=LINE_LENGTH) :: solutionFileName
4145
integer, allocatable :: Nx(:), Ny(:), Nz(:)
4246
integer :: Nmax
4347

48+
4449
call SetSolver(NAVIERSTOKES_SOLVER)
4550
!
4651
! -----------------------------------------
@@ -96,21 +101,51 @@ PROGRAM HORSES3DMainNS
96101
call InitializeNodalStorage (controlVariables ,Nmax)
97102
call Initialize_InterpolationMatrices(Nmax)
98103

104+
! MPI Partition option for MLRK time integeration
105+
if ((MPI_Process % doMPIAction).and.(trim(controlVariables % stringValueForKey('explicit method', requestedLength = LINE_LENGTH)) == 'multi level rk3')) then
106+
107+
isMLRK = .TRUE. ! The time integration is Multi-Level RK
108+
109+
if ( controlVariables % ContainsKey("optimized partition") ) then
110+
optimizePartitionLevel = controlVariables % LogicalValueForKey ("optimized partition")
111+
end if
112+
113+
if (optimizePartitionLevel) generateMonitor =.FALSE. ! Do not generate monitor for the first construction of sem as it will be reconstruct
114+
end if
115+
116+
! Construct DGSEM library
99117
call sem % construct ( controlVariables = controlVariables, &
100118
Nx_ = Nx, Ny_ = Ny, Nz_ = Nz, &
101-
success = success)
102-
103-
call Initialize_SpaceAndTimeMethods(controlVariables, sem)
104-
105-
IF(.NOT. success) error stop "Mesh reading error"
106-
IF(.NOT. success) error stop "Boundary condition specification error"
107-
CALL UserDefinedFinalSetup(sem % mesh, thermodynamics, dimensionless, refValues)
119+
success = success, generateMonitor =generateMonitor)
108120
!
109121
! -------------------------
110122
! Set the initial condition
111123
! -------------------------
112124
!
113-
call sem % SetInitialCondition(controlVariables, initial_iteration, initial_time)
125+
call UserDefinedFinalSetup(sem % mesh, thermodynamics, dimensionless, refValues)
126+
call sem % SetInitialCondition(controlVariables, initial_iteration, initial_time)
127+
!
128+
! ----------------------------------------------
129+
! Reconstruct for MLRK explicit time step method
130+
! ----------------------------------------------
131+
!
132+
if (isMLRK .and. optimizePartitionLevel .and. MPI_Process % doMPIAction) then
133+
call sem % reconstruct ( controlVariables = controlVariables, &
134+
Nx_ = Nx, Ny_ = Ny, Nz_ = Nz, &
135+
success = success)
136+
call UserDefinedFinalSetup(sem % mesh, thermodynamics, dimensionless, refValues)
137+
call sem % SetInitialCondition(controlVariables, initial_iteration, initial_time)
138+
end if
139+
!
140+
! -----------------------------
141+
! Initialize the discretization
142+
! -----------------------------
143+
!
144+
call Initialize_SpaceAndTimeMethods(controlVariables, sem)
145+
146+
IF(.NOT. success) error stop "Mesh reading error"
147+
IF(.NOT. success) error stop "Boundary condition specification error"
148+
114149
!
115150
! -------------------
116151
! Build the particles

0 commit comments

Comments
 (0)