Skip to content

Reduce threading barriers in ocean #3

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: azamat/ocean/fix-bgc-threading
Choose a base branch
from
Open

Reduce threading barriers in ocean #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0c12a20
Update ocean BGC threading
amametjanov Feb 7, 2019
8838e74
Reduce threading barriers in ocean
amametjanov Feb 13, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 0 additions & 4 deletions src/core_ocean/mode_forward/mpas_ocn_time_integration_rk4.F
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -439,7 +439,6 @@ subroutine ocn_time_integrator_rk4(domain, dt)!{{{
* highFreqThicknessTend(:, iCell)
end do
!$omp end do
call mpas_threading_barrier()
block => block % next
end do

Expand Down Expand Up @@ -642,7 +641,6 @@ subroutine ocn_time_integrator_rk4(domain, dt)!{{{
normalTransportVelocity(:, iEdge) = normalVelocityNew(:, iEdge)
end do
!$omp end do
call mpas_threading_barrier()

! Compute normalGMBolusVelocity and the tracer transport velocity
if (config_use_standardGM) then
Expand Down Expand Up @@ -1161,7 +1159,6 @@ subroutine ocn_time_integrator_rk4_diagnostic_update(block, dt, rkWeight, err)!{
normalTransportVelocity(:, iEdge) = normalVelocityProvis(:, iEdge)
end do
!$omp end do
call mpas_threading_barrier()

! Compute normalGMBolusVelocity, relativeSlope and RediDiffVertCoef if respective flags are turned on
if (config_use_standardGM) then
Expand All @@ -1176,7 +1173,6 @@ subroutine ocn_time_integrator_rk4_diagnostic_update(block, dt, rkWeight, err)!{
end do
!$omp end do
end if
call mpas_threading_barrier()
! ------------------------------------------------------------------
! End: Accumulating various parametrizations of the transport velocity
! ------------------------------------------------------------------
Expand Down
38 changes: 5 additions & 33 deletions src/core_ocean/mode_forward/mpas_ocn_time_integration_split.F
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -397,7 +397,6 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{

call mpas_pool_get_subpool(domain % blocklist % structs, 'diagnostics', diagnosticsPool)

call mpas_threading_barrier()
! --- update halos for diagnostic ocean boundary layer depth
if (config_use_cvmix_kpp) then
call mpas_timer_start("se halo diag obd")
Expand Down Expand Up @@ -438,15 +437,12 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{
end do
call mpas_timer_stop("se freq-filtered-thick computations")

call mpas_threading_barrier()

call mpas_timer_start("se freq-filtered-thick halo update")

call mpas_dmpar_field_halo_exch(domain, 'tendHighFreqThickness')
call mpas_dmpar_field_halo_exch(domain, 'tendLowFreqDivergence')

call mpas_timer_stop("se freq-filtered-thick halo update")
call mpas_threading_barrier()

block => domain % blocklist
do while (associated(block))
Expand Down Expand Up @@ -560,7 +556,7 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{

allocate(uTemp(nVertLevels))

!$omp do schedule(runtime) private(cell1, cell2, k, normalThicknessFluxSum, thicknessSum)
!$omp do schedule(runtime)
do iEdge = 1, nEdges
cell1 = cellsOnEdge(1,iEdge)
cell2 = cellsOnEdge(2,iEdge)
Expand Down Expand Up @@ -609,8 +605,6 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{
block => block % next
end do

call mpas_threading_barrier()

call mpas_timer_start("se halo normalBaroclinicVelocity")
call mpas_dmpar_field_halo_exch(domain, 'normalBaroclinicVelocity', timeLevel=2)
call mpas_timer_stop("se halo normalBaroclinicVelocity")
Expand Down Expand Up @@ -761,21 +755,16 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{
edgeHaloComputeCounter = 0
neededHalos = 1 + config_n_btr_cor_iter

call mpas_threading_barrier()

call mpas_timer_start('btr se subcycle loop')
do j = 1, nBtrSubcycles * config_btr_subcycle_loop_factor
if(cellHaloComputeCounter < neededHalos) then

call mpas_threading_barrier()
call mpas_timer_start('se halo subcycle')
call mpas_dmpar_exch_group_reuse_halo_exch(domain, subcycleGroupName, timeLevel=oldBtrSubcycleTime)
call mpas_threading_barrier()
call mpas_timer_stop('se halo subcycle')

cellHaloComputeCounter = config_num_halos - mod( config_num_halos, neededHalos )
edgeHaloComputeCounter = config_num_halos + 1 - mod( config_num_halos, neededHalos )
call mpas_threading_barrier()
end if

!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
Expand Down Expand Up @@ -813,7 +802,7 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{
nEdges = nEdgesPtr
nEdges = nEdgesArray( edgeHaloComputeCounter )

!$omp do schedule(runtime) private(cell1, cell2, CoriolisTerm, i, eoe)
!$omp do schedule(runtime)
do iEdge = 1, nEdges

temp_mask = edgeMask(1, iEdge)
Expand Down Expand Up @@ -1025,7 +1014,7 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{

nEdges = nEdgesArray( edgeHaloComputeCounter )

!$omp do schedule(runtime) private(cell1, cell2, eoe, CoriolisTerm, i, sshCell1, sshCell2)
!$omp do schedule(runtime)
do iEdge = 1, nEdges

! asarje: added to avoid redundant computations based on mask
Expand Down Expand Up @@ -1235,7 +1224,6 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{
end do ! j=1,nBtrSubcycles
call mpas_timer_stop('btr se subcycle loop')

call mpas_threading_barrier()
call mpas_pool_get_subpool(domain % blocklist % structs, 'scratch', scratchPool)
call mpas_pool_get_field(scratchPool, 'btrvel_temp', btrvel_tempField)
call mpas_deallocate_scratch_field(btrvel_tempField, .false.)
Expand Down Expand Up @@ -1276,8 +1264,6 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{
end do ! block
call mpas_timer_stop('btr se norm')

call mpas_threading_barrier()

! boundary update on F
call mpas_timer_start("se halo F and btr vel")
call mpas_dmpar_exch_group_create(domain, finalBtrGroupName)
Expand All @@ -1287,12 +1273,11 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{

call mpas_threading_barrier()
call mpas_dmpar_exch_group_full_halo_exch(domain, finalBtrGroupName)
call mpas_threading_barrier()

call mpas_dmpar_exch_group_destroy(domain, finalBtrGroupName)
call mpas_timer_stop("se halo F and btr vel")

call mpas_threading_barrier()

! Check that you can compute SSH using the total sum or the individual increments
! over the barotropic subcycles.
! efficiency: This next block of code is really a check for debugging, and can
Expand Down Expand Up @@ -1338,7 +1323,7 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{
useVelocityCorrection = 0
endif

!$omp do schedule(runtime) private(k, normalThicknessFluxSum, thicknessSum, normalVelocityCorrection)
!$omp do schedule(runtime)
do iEdge = 1, nEdges

! velocity for normalVelocityCorrectionection is normalBarotropicVelocity + normalBaroclinicVelocity + uBolus
Expand Down Expand Up @@ -1443,17 +1428,13 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{
end do
call mpas_timer_stop('se thick tend')

call mpas_threading_barrier()

! update halo for thickness tendencies
call mpas_timer_start("se halo thickness")

call mpas_dmpar_field_halo_exch(domain, 'tendLayerThickness')

call mpas_timer_stop("se halo thickness")

call mpas_threading_barrier()

call mpas_timer_start('se tracer tend', .false.)
block => domain % blocklist
do while (associated(block))
Expand All @@ -1473,8 +1454,6 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{
end do
call mpas_timer_stop('se tracer tend')

call mpas_threading_barrier()

! update halo for tracer tendencies
call mpas_timer_start("se halo tracers")
call mpas_pool_get_subpool(domain % blocklist % structs, 'tend', tendPool)
Expand All @@ -1491,8 +1470,6 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{
end do
call mpas_timer_stop("se halo tracers")

call mpas_threading_barrier()

call mpas_timer_start('se loop fini')
block => domain % blocklist
do while (associated(block))
Expand Down Expand Up @@ -1807,8 +1784,6 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{
block => block % next
end do

call mpas_threading_barrier()

! Update halo on u and tracers, which were just updated for implicit vertical mixing. If not done,
! this leads to lack of volume conservation. It is required because halo updates in stage 3 are only
! conducted on tendencies, not on the velocity and tracer fields. So this update is required to
Expand All @@ -1832,8 +1807,6 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{

call mpas_timer_stop("se implicit vert mix")

call mpas_threading_barrier()

call mpas_timer_start('se fini')
block => domain % blocklist
do while (associated(block))
Expand Down Expand Up @@ -1930,7 +1903,6 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{
!$omp end do

call ocn_time_average_coupled_accumulate(diagnosticsPool, statePool, forcingPool, 2)
call mpas_threading_barrier()

if (config_use_standardGM) then
call ocn_reconstruct_gm_vectors(diagnosticsPool, meshPool)
Expand Down
Loading