Merge branch 'atmosphere/acc_atm_get_bdy_states' into develop (PR #1279)

This merge adds OpenACC directives and data movement to the
mpas_atm_get_bdy_state_2d and mpas_atm_get_bdy_state_3d functions.

Timing information for the OpenACC data transfers in these routines is captured in
the log file by new timers:

 - mpas_atm_get_bdy_state_2d [ACC_data_xfer].
 - mpas_atm_get_bdy_state_3d [ACC_data_xfer].

* atmosphere/acc_atm_get_bdy_states:
  Enforce correct OpenACC data in mpas_atm_get_bdy_state_{2d,3d} functions
  Add OpenACC data movement to mpas_atm_get_bdy_state_{2d,3d} functions
  Initial OpenACC port of the mpas_atm_get_bdy_state{2d,3d} functions

Author: mgduda

src/core_atmosphere/dynamics/mpas_atm_boundaries.F

@@ -7,6 +7,15 @@
 !
 module mpas_atm_boundaries
 
+#ifdef MPAS_OPENACC
+    use mpas_timer, only: mpas_timer_start, mpas_timer_stop
+#define MPAS_ACC_TIMER_START(X) call mpas_timer_start(X)
+#define MPAS_ACC_TIMER_STOP(X) call mpas_timer_stop(X)
+#else
+#define MPAS_ACC_TIMER_START(X)
+#define MPAS_ACC_TIMER_STOP(X)
+#endif
+
     use mpas_derived_types, only : mpas_pool_type, mpas_clock_type, block_type, mpas_time_type, mpas_timeInterval_type, MPAS_NOW, &
                                    MPAS_STREAM_LATEST_BEFORE, MPAS_STREAM_EARLIEST_STRICTLY_AFTER, &
                                    MPAS_streamManager_type
@@ -363,7 +372,7 @@ function mpas_atm_get_bdy_state_2d(clock, block, vertDim, horizDim, field, delta
         real (kind=RKIND), dimension(vertDim,horizDim+1) :: return_state
 
         type (mpas_pool_type), pointer :: lbc
-        integer, pointer :: idx
+        integer, pointer :: idx_ptr
         real (kind=RKIND), dimension(:,:), pointer :: tend
         real (kind=RKIND), dimension(:,:), pointer :: state
         real (kind=RKIND), dimension(:,:,:), pointer :: tend_scalars
@@ -374,6 +383,7 @@ function mpas_atm_get_bdy_state_2d(clock, block, vertDim, horizDim, field, delta
         real (kind=RKIND) :: dt
         integer :: err_level
         integer :: ierr
+        integer :: i,j,idx
 
 
         currTime = mpas_get_clock_time(clock, MPAS_NOW, ierr)
@@ -410,13 +420,49 @@ function mpas_atm_get_bdy_state_2d(clock, block, vertDim, horizDim, field, delta
         ! query the field as a scalar constituent
         !
         if (associated(tend) .and. associated(state)) then
-            return_state(:,:) = state(:,:) - dt * tend(:,:)
+            MPAS_ACC_TIMER_START('mpas_atm_get_bdy_state_2d [ACC_data_xfer]')
+            !$acc enter data create(return_state) &
+            !$acc            copyin(tend, state)
+            MPAS_ACC_TIMER_STOP('mpas_atm_get_bdy_state_2d [ACC_data_xfer]')
+
+            !$acc parallel default(present)
+            !$acc loop gang vector collapse(2)
+            do i=1, horizDim+1
+                do j=1, vertDim
+                    return_state(j,i) = state(j,i) - dt * tend(j,i)
+                end do
+            end do
+            !$acc end parallel
+
+            MPAS_ACC_TIMER_START('mpas_atm_get_bdy_state_2d [ACC_data_xfer]')
+            !$acc exit data copyout(return_state) &
+            !$acc           delete(tend, state)
+            MPAS_ACC_TIMER_STOP('mpas_atm_get_bdy_state_2d [ACC_data_xfer]')
         else
             call mpas_pool_get_array(lbc, 'lbc_scalars', tend_scalars, 1)
             call mpas_pool_get_array(lbc, 'lbc_scalars', state_scalars, 2)
-            call mpas_pool_get_dimension(lbc, 'index_'//trim(field), idx)
+            call mpas_pool_get_dimension(lbc, 'index_'//trim(field), idx_ptr)
+
+            idx=idx_ptr ! Avoid non-array pointer for OpenACC
 
-            return_state(:,:) = state_scalars(idx,:,:) - dt * tend_scalars(idx,:,:)
+            MPAS_ACC_TIMER_START('mpas_atm_get_bdy_state_2d [ACC_data_xfer]')
+            !$acc enter data create(return_state) &
+            !$acc            copyin(tend_scalars, state_scalars)
+            MPAS_ACC_TIMER_STOP('mpas_atm_get_bdy_state_2d [ACC_data_xfer]')
+
+            !$acc parallel default(present)
+            !$acc loop gang vector collapse(2)
+            do i=1, horizDim+1
+                do j=1, vertDim
+                    return_state(j,i) = state_scalars(idx,j,i) - dt * tend_scalars(idx,j,i)
+                end do
+            end do
+            !$acc end parallel
+
+            MPAS_ACC_TIMER_START('mpas_atm_get_bdy_state_2d [ACC_data_xfer]')
+            !$acc exit data copyout(return_state) &
+            !$acc           delete(tend_scalars, state_scalars)
+            MPAS_ACC_TIMER_STOP('mpas_atm_get_bdy_state_2d [ACC_data_xfer]')
         end if
 
     end function mpas_atm_get_bdy_state_2d
@@ -476,6 +522,7 @@ function mpas_atm_get_bdy_state_3d(clock, block, innerDim, vertDim, horizDim, fi
         real (kind=RKIND) :: dt
         integer :: err_level
         integer :: ierr
+        integer :: i,j,k
 
 
         currTime = mpas_get_clock_time(clock, MPAS_NOW, ierr)
@@ -496,7 +543,26 @@ function mpas_atm_get_bdy_state_3d(clock, block, innerDim, vertDim, horizDim, fi
         call mpas_pool_get_array(lbc, 'lbc_'//trim(field), tend, 1)
         call mpas_pool_get_array(lbc, 'lbc_'//trim(field), state, 2)
 
-        return_state(:,:,:) = state(:,:,:) - dt * tend(:,:,:)
+        MPAS_ACC_TIMER_START('mpas_atm_get_bdy_state_3d [ACC_data_xfer]')
+        !$acc enter data create(return_state) &
+        !$acc            copyin(tend, state)
+        MPAS_ACC_TIMER_STOP('mpas_atm_get_bdy_state_3d [ACC_data_xfer]')
+
+        !$acc parallel default(present)
+        !$acc loop gang vector collapse(3)
+        do i=1, horizDim+1
+            do j=1, vertDim
+                do k=1, innerDim
+                    return_state(k,j,i) = state(k,j,i) - dt * tend(k,j,i)
+                end do
+            end do
+        end do
+        !$acc end parallel
+
+        MPAS_ACC_TIMER_START('mpas_atm_get_bdy_state_3d [ACC_data_xfer]')
+        !$acc exit data copyout(return_state) &
+        !$acc           delete(tend, state)
+        MPAS_ACC_TIMER_STOP('mpas_atm_get_bdy_state_3d [ACC_data_xfer]')
 
     end function mpas_atm_get_bdy_state_3d