Add acc data movement to atm_advance_acoustic_step_work

Ensure that the other, non-invariant fields are available for this
routine. Variables that are overwritten during this routine are only
created while others are copied in. Any variables overwritten by this
routine are copied out at the end.

Timing for these transfers are reported in the output log file in the
new timer: 'atm_advance_acoustic_step [ACC_data_xfer]'.

Author: gdicker1

src/core_atmosphere/dynamics/mpas_atm_time_integration.F

@@ -2394,6 +2394,18 @@ subroutine atm_advance_acoustic_step_work(nCells, nEdges, nCellsSolve, cellStart
       resm = (1.0 - epssm) / (1.0 + epssm)
       rdts = 1./dts
 
+      MPAS_ACC_TIMER_START('atm_advance_acoustic_step [ACC_data_xfer]')
+      !$acc enter data copyin(exner,cqu,cofwt,coftz,cofrz,cofwr,cofwz, &
+      !$acc                a_tri,alpha_tri,gamma_tri,rho_zz,theta_m,w, &
+      !$acc                tend_ru,tend_rho,tend_rt,tend_rw,rw,rw_save)
+      !$acc enter data create(rtheta_pp_old)
+      if(small_step == 1) then
+         !$acc enter data create(ru_p,ruAvg,rho_pp,rtheta_pp,wwAvg,rw_p)
+      else
+         !$acc enter data copyin(ru_p,ruAvg,rho_pp,rtheta_pp,wwAvg,rw_p)
+      end if
+      MPAS_ACC_TIMER_STOP('atm_advance_acoustic_step [ACC_data_xfer]')
+
       if(small_step /= 1) then  !  not needed on first small step
 
         ! forward-backward acoustic step integration.
@@ -2611,6 +2623,13 @@ subroutine atm_advance_acoustic_step_work(nCells, nEdges, nCellsSolve, cellStart
       end do !  end of loop over cells
       !$acc end parallel
 
+      MPAS_ACC_TIMER_START('atm_advance_acoustic_step [ACC_data_xfer]')
+      !$acc exit data delete(exner,cqu,cofwt,coftz,cofrz,cofwr,cofwz, &
+      !$acc                a_tri,alpha_tri,gamma_tri,rho_zz,theta_m,w, &
+      !$acc                tend_ru,tend_rho,tend_rt,tend_rw,rw,rw_save)
+      !$acc exit data copyout(rtheta_pp_old,ru_p,ruAvg,rho_pp, &
+      !$acc                   rtheta_pp,wwAvg,rw_p)
+      MPAS_ACC_TIMER_STOP('atm_advance_acoustic_step [ACC_data_xfer]')
 
    end subroutine atm_advance_acoustic_step_work