Skip to content

Commit c3aee17

Browse files
abishekg7abishekg7
committed
initial OpenACC port of atm_rk_dynamics_substep_finish
1 parent 5cb5c63 commit c3aee17

File tree

1 file changed

+105
-21
lines changed

1 file changed

+105
-21
lines changed

‎src/core_atmosphere/dynamics/mpas_atm_time_integration.F

+105-21
Original file line numberDiff line numberDiff line change
@@ -761,9 +761,9 @@ subroutine atm_srk3(domain, dt, itimestep, exchange_halo_group)
761761
#ifdef DO_PHYSICS
762762
call mpas_pool_get_dimension(state, 'index_qv', index_qv)
763763
#endif
764-
if (config_apply_lbcs) then
765-
call mpas_pool_get_dimension(state, 'num_scalars', num_scalars)
766-
endif
764+
if (config_apply_lbcs) then
765+
call mpas_pool_get_dimension(state, 'num_scalars', num_scalars)
766+
endif
767767

768768
!
769769
! allocate storage for physics tendency save
@@ -1287,7 +1287,7 @@ subroutine atm_srk3(domain, dt, itimestep, exchange_halo_group)
12871287

12881288
!$OMP PARALLEL DO
12891289
do thread=1,nThreads
1290-
call atm_rk_dynamics_substep_finish(state, diag, dynamics_substep, dynamics_split, &
1290+
call atm_rk_dynamics_substep_finish(state, diag, nVertLevels, dynamics_substep, dynamics_split, &
12911291
cellThreadStart(thread), cellThreadEnd(thread), &
12921292
vertexThreadStart(thread), vertexThreadEnd(thread), &
12931293
edgeThreadStart(thread), edgeThreadEnd(thread), &
@@ -6200,7 +6200,7 @@ subroutine atm_init_coupled_diagnostics(state, time_lev, diag, mesh, configs, &
62006200
end subroutine atm_init_coupled_diagnostics
62016201

62026202

6203-
subroutine atm_rk_dynamics_substep_finish( state, diag, dynamics_substep, dynamics_split, &
6203+
subroutine atm_rk_dynamics_substep_finish( state, diag, nVertLevels, dynamics_substep, dynamics_split, &
62046204
cellStart, cellEnd, vertexStart, vertexEnd, edgeStart, edgeEnd, &
62056205
cellSolveStart, cellSolveEnd, vertexSolveStart, vertexSolveEnd, edgeSolveStart, edgeSolveEnd)
62066206

@@ -6214,7 +6214,7 @@ subroutine atm_rk_dynamics_substep_finish( state, diag, dynamics_substep, dynami
62146214

62156215
type (mpas_pool_type), intent(inout) :: state
62166216
type (mpas_pool_type), intent(inout) :: diag
6217-
integer, intent(in) :: dynamics_substep, dynamics_split
6217+
integer, intent(in) :: nVertLevels, dynamics_substep, dynamics_split
62186218
integer, intent(in) :: cellStart, cellEnd, vertexStart, vertexEnd, edgeStart, edgeEnd
62196219
integer, intent(in) :: cellSolveStart, cellSolveEnd, vertexSolveStart, vertexSolveEnd, edgeSolveStart, edgeSolveEnd
62206220

@@ -6234,6 +6234,7 @@ subroutine atm_rk_dynamics_substep_finish( state, diag, dynamics_substep, dynami
62346234
real (kind=RKIND), dimension(:,:), pointer :: theta_m_1, theta_m_2
62356235
real (kind=RKIND), dimension(:,:), pointer :: rho_zz_1, rho_zz_2, rho_zz_old_split
62366236
real (kind=RKIND), dimension(:,:), pointer :: ruAvg, wwAvg, ruAvg_split, wwAvg_split
6237+
integer :: iCell, iEdge, j, k
62376238

62386239
call mpas_pool_get_array(diag, 'ru', ru)
62396240
call mpas_pool_get_array(diag, 'ru_save', ru_save)
@@ -6258,35 +6259,118 @@ subroutine atm_rk_dynamics_substep_finish( state, diag, dynamics_substep, dynami
62586259
call mpas_pool_get_array(state, 'rho_zz', rho_zz_1, 1)
62596260
call mpas_pool_get_array(state, 'rho_zz', rho_zz_2, 2)
62606261

6262+
6263+
MPAS_ACC_TIMER_START('atm_rk_dynamics_substep_finish [ACC_data_xfer]')
6264+
!$acc enter data create(ru_save, u_1, rtheta_p_save, theta_m_1, rho_p_save, rw_save, &
6265+
!$acc w_1, rho_zz_1) &
6266+
!$acc copyin(ru, u_2, rtheta_p, rho_p, theta_m_2, rho_zz_2, rw, &
6267+
!$acc w_2, ruAvg, wwAvg, ruAvg_split, wwAvg_split, rho_zz_old_split)
6268+
MPAS_ACC_TIMER_STOP('atm_rk_dynamics_substep_finish [ACC_data_xfer]')
6269+
62616270
inv_dynamics_split = 1.0_RKIND / real(dynamics_split)
62626271

6272+
!$acc parallel
62636273
if (dynamics_substep < dynamics_split) then
62646274

6265-
ru_save(:,edgeStart:edgeEnd) = ru(:,edgeStart:edgeEnd)
6266-
rw_save(:,cellStart:cellEnd) = rw(:,cellStart:cellEnd)
6267-
rtheta_p_save(:,cellStart:cellEnd) = rtheta_p(:,cellStart:cellEnd)
6268-
rho_p_save(:,cellStart:cellEnd) = rho_p(:,cellStart:cellEnd)
6275+
!$acc loop gang worker
6276+
do iEdge = edgeStart,edgeEnd
6277+
!$acc loop vector
6278+
do k = 1,nVertLevels
6279+
ru_save(k,iEdge) = ru(k,iEdge)
6280+
u_1(k,iEdge) = u_2(k,iEdge)
6281+
end do
6282+
end do
6283+
62696284

6270-
u_1(:,edgeStart:edgeEnd) = u_2(:,edgeStart:edgeEnd)
6271-
w_1(:,cellStart:cellEnd) = w_2(:,cellStart:cellEnd)
6272-
theta_m_1(:,cellStart:cellEnd) = theta_m_2(:,cellStart:cellEnd)
6273-
rho_zz_1(:,cellStart:cellEnd) = rho_zz_2(:,cellStart:cellEnd)
6285+
!$acc loop gang worker
6286+
do iCell = cellStart,cellEnd
6287+
!$acc loop vector
6288+
do k = 1,nVertLevels
6289+
rtheta_p_save(k,iCell) = rtheta_p(k,iCell)
6290+
rho_p_save(k,iCell) = rho_p(k,iCell)
6291+
theta_m_1(k,iCell) = theta_m_2(k,iCell)
6292+
rho_zz_1(k,iCell) = rho_zz_2(k,iCell)
6293+
end do
6294+
end do
62746295

6296+
!$acc loop gang worker
6297+
do iCell = cellStart,cellEnd
6298+
!$acc loop vector
6299+
do k = 1,nVertLevels+1
6300+
rw_save(k,iCell) = rw(k,iCell)
6301+
w_1(k,iCell) = w_2(k,iCell)
6302+
end do
6303+
end do
62756304
end if
62766305

6306+
6307+
!!$acc parallel
62776308
if (dynamics_substep == 1) then
6278-
ruAvg_split(:,edgeStart:edgeEnd) = ruAvg(:,edgeStart:edgeEnd)
6279-
wwAvg_split(:,cellStart:cellEnd) = wwAvg(:,cellStart:cellEnd)
6309+
!$acc loop gang worker
6310+
do iEdge = edgeStart,edgeEnd
6311+
!$acc loop vector
6312+
do k = 1,nVertLevels
6313+
ruAvg_split(k,iEdge) = ruAvg(k,iEdge)
6314+
end do
6315+
end do
6316+
!$acc loop gang worker
6317+
do iCell = cellStart,cellEnd
6318+
!$acc loop vector
6319+
do k = 1,nVertLevels+1
6320+
wwAvg_split(k,iCell) = wwAvg(k,iCell)
6321+
end do
6322+
end do
62806323
else
6281-
ruAvg_split(:,edgeStart:edgeEnd) = ruAvg(:,edgeStart:edgeEnd)+ruAvg_split(:,edgeStart:edgeEnd)
6282-
wwAvg_split(:,cellStart:cellEnd) = wwAvg(:,cellStart:cellEnd)+wwAvg_split(:,cellStart:cellEnd)
6324+
!$acc loop gang worker
6325+
do iEdge = edgeStart,edgeEnd
6326+
!$acc loop vector
6327+
do k = 1,nVertLevels
6328+
ruAvg_split(k,iEdge) = ruAvg(k,iEdge) + ruAvg_split(k,iEdge)
6329+
end do
6330+
end do
6331+
!$acc loop gang worker
6332+
do iCell = cellStart,cellEnd
6333+
!$acc loop vector
6334+
do k = 1,nVertLevels+1
6335+
wwAvg_split(k,iCell) = wwAvg(k,iCell) + wwAvg_split(k,iCell)
6336+
end do
6337+
end do
62836338
end if
62846339

62856340
if (dynamics_substep == dynamics_split) then
6286-
ruAvg(:,edgeStart:edgeEnd) = ruAvg_split(:,edgeStart:edgeEnd) * inv_dynamics_split
6287-
wwAvg(:,cellStart:cellEnd) = wwAvg_split(:,cellStart:cellEnd) * inv_dynamics_split
6288-
rho_zz_1(:,cellStart:cellEnd) = rho_zz_old_split(:,cellStart:cellEnd)
6341+
!$acc loop gang worker
6342+
do iEdge = edgeStart,edgeEnd
6343+
!$acc loop vector
6344+
do k = 1,nVertLevels
6345+
ruAvg(k,iEdge) = ruAvg_split(k,iEdge) * inv_dynamics_split
6346+
end do
6347+
end do
6348+
!$acc loop gang worker
6349+
do iCell = cellStart,cellEnd
6350+
!$acc loop vector
6351+
do k = 1,nVertLevels+1
6352+
wwAvg(k,iCell) = wwAvg_split(k,iCell) * inv_dynamics_split
6353+
end do
6354+
end do
6355+
!$acc loop gang worker
6356+
do iCell = cellStart,cellEnd
6357+
!$acc loop vector
6358+
do k = 1,nVertLevels
6359+
rho_zz_1(k,iCell) = rho_zz_old_split(k,iCell)
6360+
end do
6361+
end do
62896362
end if
6363+
!$acc end parallel
6364+
6365+
!!$acc exit data copyout(rho_zz_1, ruAvg, wwAvg, ruAvg_split, wwAvg_split)
6366+
6367+
MPAS_ACC_TIMER_START('atm_rk_dynamics_substep_finish [ACC_data_xfer]')
6368+
!$acc exit data copyout(ru_save, u_1, rtheta_p_save, rho_p_save, rw_save, &
6369+
!$acc w_1, theta_m_1, rho_zz_1, ruAvg, wwAvg, ruAvg_split, &
6370+
!$acc wwAvg_split) &
6371+
!$acc delete(ru, u_2, rtheta_p, rho_p, theta_m_2, rho_zz_2, rw, &
6372+
!$acc w_2, rho_zz_old_split)
6373+
MPAS_ACC_TIMER_STOP('atm_rk_dynamics_substep_finish [ACC_data_xfer]')
62906374

62916375
end subroutine atm_rk_dynamics_substep_finish
62926376

0 commit comments

Comments
 (0)