Add OpenACC directives to enable executing mpas_atm_get_bdy_tend on GPUs.

Note this commit adds "mpas_atm_get_bdy_tend [ACC_data_xfer]" timers to
time the data transfers done in mpas_atm_get_bdy_tend, but there is no
timer for the actual computation done in mpas_atm_get_bdy_tend.

Author: jim-p-w

src/core_atmosphere/dynamics/mpas_atm_boundaries.F

@@ -7,6 +7,15 @@
 !
 module mpas_atm_boundaries
 
+#ifdef MPAS_OPENACC
+use mpas_timer
+#define MPAS_ACC_TIMER_START(X) call mpas_timer_start(X)
+#define MPAS_ACC_TIMER_STOP(X) call mpas_timer_stop(X)
+#else
+#define MPAS_ACC_TIMER_START(X)
+#define MPAS_ACC_TIMER_STOP(X)
+#endif
+
     use mpas_derived_types, only : mpas_pool_type, mpas_clock_type, block_type, mpas_time_type, mpas_timeInterval_type, MPAS_NOW, &
                                    MPAS_STREAM_LATEST_BEFORE, MPAS_STREAM_EARLIEST_STRICTLY_AFTER, &
                                    MPAS_streamManager_type
@@ -291,25 +300,57 @@ function mpas_atm_get_bdy_tend(clock, block, vertDim, horizDim, field, delta_t)
         real (kind=RKIND), dimension(vertDim,horizDim+1) :: return_tend
 
         type (mpas_pool_type), pointer :: lbc
-        integer, pointer :: idx
+        integer, pointer :: idx_ptr
         real (kind=RKIND), dimension(:,:), pointer :: tend
         real (kind=RKIND), dimension(:,:,:), pointer :: tend_scalars
-        integer :: ierr
+        integer :: idx, i, j
 
 
         call mpas_pool_get_subpool(block % structs, 'lbc', lbc)
 
         nullify(tend)
         call mpas_pool_get_array(lbc, 'lbc_'//trim(field), tend, 1)
 
+        MPAS_ACC_TIMER_START('mpas_atm_get_bdy_tend [ACC_data_xfer]')
+        !$acc enter data create(return_tend)
         if (associated(tend)) then
-            return_tend(:,:) = tend(:,:)
+            !$acc enter data copyin(tend)
         else
             call mpas_pool_get_array(lbc, 'lbc_scalars', tend_scalars, 1)
-            call mpas_pool_get_dimension(lbc, 'index_'//trim(field), idx)
+            !$acc enter data copyin(tend_scalars)
+
+            ! Ensure the integer pointed to by idx_ptr is copied to the gpu device
+            call mpas_pool_get_dimension(lbc, 'index_'//trim(field), idx_ptr)
+            idx = idx_ptr
+        end if
+        MPAS_ACC_TIMER_STOP('mpas_atm_get_bdy_tend [ACC_data_xfer]')
 
-            return_tend(:,:) = tend_scalars(idx,:,:)
+        !$acc parallel default(present)
+        if (associated(tend)) then
+            !$acc loop gang vector collapse(2)
+            do j=1,horizDim+1
+                do i=1,vertDim
+                    return_tend(i,j) = tend(i,j)
+                end do
+            end do
+        else
+            !$acc loop vector collapse(2)
+            do j=1,horizDim+1
+                do i=1,vertDim
+                    return_tend(i,j) = tend_scalars(idx,i,j)
+                end do
+            end do
+        end if
+        !$acc end parallel
+
+        MPAS_ACC_TIMER_START('mpas_atm_get_bdy_tend [ACC_data_xfer]')
+        !$acc exit data copyout(return_tend)
+        if (associated(tend)) then
+            !$acc exit data delete(tend)
+        else
+            !$acc exit data delete(tend_scalars)
         end if
+        MPAS_ACC_TIMER_STOP('mpas_atm_get_bdy_tend [ACC_data_xfer]')
 
     end function mpas_atm_get_bdy_tend