Skip to content

Commit 2fa29ed

Browse files
uturuncogluuturuncoglu
committed
(regesm) : mods for COP to add halo regions
1 parent 9df6454 commit 2fa29ed

File tree

3 files changed

+125
-50
lines changed

3 files changed

+125
-50
lines changed

‎mod_esmf_cop.F90‎

Lines changed: 102 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,9 @@ module mod_esmf_cop
4242
integer :: nports
4343
character(255), allocatable :: input_ports(:)
4444
character(ESMF_MAXSTR), allocatable, save :: gridnames(:)
45+
!
46+
integer :: tuw(3), tlw(3)
47+
logical :: hasRight, hasLeft, hasTop, hasBottom
4548
!
4649
type(ESMF_RouteHandle), allocatable :: routeHandle(:)
4750
!
@@ -514,11 +517,10 @@ subroutine COP_SetInitializeP5(gcomp, &
514517
!-----------------------------------------------------------------------
515518
!
516519
integer :: i, j, k, localPet, rank, itemCount, localDECount
517-
integer :: tileX, tileY, tuw(3)
520+
integer :: tileX, tileY
518521
character(ESMF_MAXSTR), allocatable :: itemNameList(:)
519522
real(ESMF_KIND_R8), pointer :: ptr2d(:,:)
520523
real(ESMF_KIND_R8), pointer :: ptr3d(:,:,:)
521-
logical :: hasRight, hasTop
522524
!
523525
type(ESMF_VM) :: vm
524526
type(ESMF_Field) :: field
@@ -579,16 +581,26 @@ subroutine COP_SetInitializeP5(gcomp, &
579581
!
580582
hasRight = .false.
581583
if ((localPet/tileY+1) < tileX) hasRight = .true.
584+
!
585+
hasLeft = .false.
586+
if ((localPet/tileY+1) > 1) hasLeft = .true.
582587
!
583588
hasTop = .false.
584589
if (mod(localPet+1,tileY) /= 0) hasTop = .true.
590+
!
591+
hasBottom = .false.
592+
if (mod(localPet,tileY) > 0) hasBottom = .true.
585593
!
586594
tuw = 0
587-
if (hasTop) tuw(1) = 1
588-
if (hasRight) tuw(2) = 1
595+
tlw = 0
596+
if (hasTop) tuw(1) = models(Icopro)%haloWidth
597+
if (hasRight) tuw(2) = models(Icopro)%haloWidth
598+
if (hasBottom) tlw(1) = models(Icopro)%haloWidth
599+
if (hasLeft) tlw(2) = models(Icopro)%haloWidth
589600
!
590601
call ESMF_FieldEmptyComplete(field, &
591602
typekind=ESMF_TYPEKIND_R8, &
603+
totalLWidth=tlw(:rank), &
592604
totalUWidth=tuw(:rank), &
593605
rc=rc)
594606
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
@@ -757,14 +769,14 @@ subroutine COP_ModelAdvance(gcomp, rc)
757769
! Local variable declarations
758770
!-----------------------------------------------------------------------
759771
!
760-
integer :: i, j, k, its, tileX, tileY
772+
logical :: flag
773+
integer :: i, j, k, its, tileX, tileY, rank
761774
integer :: localPet, petCount, itemCount, dimCount, tileCount
762775
integer :: nPoints2D, nPoints3D
763776
integer :: cc2d(2), cc3d(3), lb(3), ub(3), dims(3)
764777
integer, allocatable :: minIndexPTile(:,:), maxIndexPTile(:,:)
765-
character(ESMF_MAXSTR) :: gname, rname, str1, str2
778+
character(ESMF_MAXSTR) :: ofile, gname, rname, str1, str2
766779
character(ESMF_MAXSTR), allocatable :: itemNameList(:)
767-
logical :: hasRight, hasTop, flag
768780
real(ESMF_KIND_R8) :: stime, etime, dtime
769781
real(ESMF_KIND_R8), dimension(:,:), pointer :: ptr2X
770782
real(ESMF_KIND_R8), dimension(:,:), pointer :: ptr2Y
@@ -782,6 +794,7 @@ subroutine COP_ModelAdvance(gcomp, rc)
782794
real(ESMF_KIND_R8), dimension(:,:), pointer :: ptr2d
783795
real(ESMF_KIND_R8), dimension(:,:,:), pointer :: ptr3d
784796
real(ESMF_KIND_R8), allocatable, dimension(:) :: var1d
797+
real(ESMF_KIND_R8), dimension(3) :: low, upp
785798
!
786799
type(ESMF_VM) :: vm
787800
type(ESMF_Grid) :: grid
@@ -929,21 +942,6 @@ subroutine COP_ModelAdvance(gcomp, rc)
929942
line=__LINE__, file=__FILE__)) return
930943
!
931944
!-----------------------------------------------------------------------
932-
! Define right and top extents
933-
! The data will be stored as Point in VTK and it requires overlap in
934-
! top and right most cells in two dimensional decomposition
935-
!-----------------------------------------------------------------------
936-
!
937-
tileX = models(Icopro)%tile(1)
938-
tileY = models(Icopro)%tile(2)
939-
!
940-
hasRight = .false.
941-
if ((localPet/tileY+1) < tileX) hasRight = .true.
942-
!
943-
hasTop = .false.
944-
if (mod(localPet+1,tileY) /= 0) hasTop = .true.
945-
!
946-
!-----------------------------------------------------------------------
947945
! Define grid in co-processor side if it is not already defined
948946
!-----------------------------------------------------------------------
949947
!
@@ -1007,8 +1005,11 @@ subroutine COP_ModelAdvance(gcomp, rc)
10071005
!-----------------------------------------------------------------------
10081006
!
10091007
nPoints2D = 1
1010-
if (hasTop) cc2d(1) = cc2d(1)+1
1011-
if (hasRight) cc2d(2) = cc2d(2)+1
1008+
if (hasTop) cc2d(1) = cc2d(1)+tuw(1)-1
1009+
if (hasRight) cc2d(2) = cc2d(2)+tuw(2)-1
1010+
if (hasBottom) cc2d(1) = cc2d(1)+tlw(1)-1
1011+
if (hasLeft) cc2d(2) = cc2d(2)+tlw(2)-1
1012+
!
10121013
do k = 1, 2
10131014
nPoints2D = nPoints2D*cc2d(k)
10141015
end do
@@ -1020,8 +1021,12 @@ subroutine COP_ModelAdvance(gcomp, rc)
10201021
lb = (/ lbound(ptr2X,dim=1), lbound(ptr2X,dim=2), 0 /)
10211022
ub = (/ ubound(ptr2X,dim=1), ubound(ptr2X,dim=2), 0 /)
10221023
!
1023-
if (hasTop) ub(1) = ub(1)+1
1024-
if (hasRight) ub(2) = ub(2)+1
1024+
if (hasTop) ub(1) = ub(1)+tuw(1)-1
1025+
if (hasRight) ub(2) = ub(2)+tuw(2)-1
1026+
if (hasBottom) lb(1) = lb(1)-tlw(1)+1
1027+
if (hasLeft) lb(2) = lb(2)-tlw(2)+1
1028+
!
1029+
! write(*,fmt="(A,5I5,I8)") "grid 2d = ", localPet, lb(1), ub(1), lb(2), ub(2), nPoints2D
10251030
!
10261031
if (debugLevel > 1) then
10271032
if (localPet == 0) then
@@ -1183,8 +1188,11 @@ subroutine COP_ModelAdvance(gcomp, rc)
11831188
!-----------------------------------------------------------------------
11841189
!
11851190
nPoints3D = 1
1186-
if (hasRight) cc3d(2) = cc3d(2)+1
1187-
if (hasTop) cc3d(1) = cc3d(1)+1
1191+
if (hasTop) cc3d(1) = cc3d(1)+tuw(1)-1
1192+
if (hasRight) cc3d(2) = cc3d(2)+tuw(2)-1
1193+
if (hasBottom) cc3d(1) = cc3d(1)+tlw(1)-1
1194+
if (hasLeft) cc3d(2) = cc3d(2)+tlw(2)-1
1195+
!
11881196
do k = 1, 3
11891197
nPoints3D = nPoints3D*cc3d(k)
11901198
end do
@@ -1198,20 +1206,25 @@ subroutine COP_ModelAdvance(gcomp, rc)
11981206
ub = (/ ubound(ptr3X,dim=1), ubound(ptr3X,dim=2), &
11991207
ubound(ptr3X,dim=3) /)
12001208
!
1201-
if (hasRight) ub(2) = ub(2)+1
1202-
if (hasTop) ub(1) = ub(1)+1
1209+
if (hasTop) ub(1) = ub(1)+tuw(1)-1
1210+
if (hasRight) ub(2) = ub(2)+tuw(2)-1
1211+
if (hasBottom) lb(1) = lb(1)-tlw(1)+1
1212+
if (hasLeft) lb(2) = lb(2)-tlw(2)+1
1213+
!
1214+
! write(*,fmt="(A,7I5,I8)") "grid 3d = ", localPet, lb(1), ub(1), lb(2), ub(2), lb(3), ub(3), nPoints3D
12031215
!
12041216
if (debugLevel > 1) then
12051217
if (localPet == 0) then
12061218
write(*,fmt="(A)") "---------------------------------------"
12071219
write(*,fmt="(A)") trim(to_upper(gname))//" GRID DEFINITION"
12081220
write(*,fmt="(A)") "---------------------------------------"
12091221
end if
1210-
write(*,fmt="(I3,6I5,3I8,I10,3I5,2L3)") localPet, lb(1), ub(1), &
1222+
write(*,fmt="(I3,6I5,3I8,I10,3I5,4L3)") localPet, lb(1), ub(1), &
12111223
lb(2), ub(2), lb(3), ub(3), &
12121224
cc3d(1), cc3d(2), cc3d(3), nPoints3D, &
12131225
maxIndexPTile(1,1),maxIndexPTile(2,1), &
1214-
maxIndexPTile(3,1), hasRight, hasTop
1226+
maxIndexPTile(3,1), hasRight, hasLeft, &
1227+
hasTop, hasBottom
12151228
end if
12161229
!
12171230
if (.not. allocated(lon1d)) then
@@ -1220,11 +1233,11 @@ subroutine COP_ModelAdvance(gcomp, rc)
12201233
allocate(lev1d(nPoints3D))
12211234
end if
12221235
!
1223-
call ntooned_3d(lb, ub, &
1236+
call ntooned_3d(localPet, .false., lb, ub, &
12241237
lon3d(lb(1):ub(1),lb(2):ub(2),lb(3):ub(3)), lon1d)
1225-
call ntooned_3d(lb, ub, &
1238+
call ntooned_3d(localPet, .false., lb, ub, &
12261239
lat3d(lb(1):ub(1),lb(2):ub(2),lb(3):ub(3)), lat1d)
1227-
call ntooned_3d(lb, ub, &
1240+
call ntooned_3d(localPet, .false., lb, ub, &
12281241
lev3d(lb(1):ub(1),lb(2):ub(2),lb(3):ub(3)), lev1d)
12291242
!
12301243
!-----------------------------------------------------------------------
@@ -1294,7 +1307,14 @@ subroutine COP_ModelAdvance(gcomp, rc)
12941307
! Create routehandle for halo update
12951308
!-----------------------------------------------------------------------
12961309
!
1297-
call ESMF_FieldHaloStore(field, routehandle=routeHandle(j), rc=rc)
1310+
k = get_varid(models(Icopro)%importField, trim(itemNameList(i)))
1311+
rank = models(Icopro)%importField(k)%rank
1312+
!
1313+
call ESMF_FieldHaloStore(field, &
1314+
routehandle=routeHandle(j), &
1315+
haloLDepth=tlw(:rank), &
1316+
haloUDepth=tuw(:rank), &
1317+
rc=rc)
12981318
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
12991319
line=__LINE__, file=FILENAME)) return
13001320
!
@@ -1310,16 +1330,6 @@ subroutine COP_ModelAdvance(gcomp, rc)
13101330
!
13111331
gridnames(j) = trim(gname)
13121332
j = j+1
1313-
!
1314-
!-----------------------------------------------------------------------
1315-
! Debug: write out component grid in VTK format
1316-
!-----------------------------------------------------------------------
1317-
!
1318-
if (debugLevel > 1) then
1319-
call ESMF_GridWriteVTK(grid,filename="coproc_"//trim(gname),rc=rc)
1320-
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU,&
1321-
line=__LINE__, file=FILENAME)) return
1322-
end if
13231333
!
13241334
end if
13251335
!
@@ -1365,17 +1375,24 @@ subroutine COP_ModelAdvance(gcomp, rc)
13651375
!
13661376
lb = (/ lbound(ptr2d,dim=1), lbound(ptr2d,dim=2), 0 /)
13671377
ub = (/ ubound(ptr2d,dim=1), ubound(ptr2d,dim=2), 0 /)
1378+
!
1379+
if (hasTop) ub(1) = ub(1)-1
1380+
if (hasRight) ub(2) = ub(2)-1
1381+
if (hasBottom) lb(1) = lb(1)+1
1382+
if (hasLeft) lb(2) = lb(2)+1
13681383
!
13691384
nPoints2D = 1
13701385
do k = 1, 2
13711386
nPoints2D = nPoints2D*(ub(k)-lb(k)+1)
13721387
end do
1388+
!
1389+
! write(*,fmt="(A,5I5,I10)") "field 2d = ", localPet, lb(1), ub(1), lb(2), ub(2), nPoints2D
13731390
!
13741391
if (.not. allocated(var1d)) then
13751392
allocate(var1d(nPoints2D))
13761393
end if
13771394
!
1378-
call ntooned_2d(lb(1:2), ub(1:2), ptr2d, var1d)
1395+
call ntooned_2d(lb(1:2), ub(1:2), ptr2d(lb(1):ub(1),lb(2):ub(2)), var1d)
13791396
!
13801397
!-----------------------------------------------------------------------
13811398
! Add field to co-processor
@@ -1423,23 +1440,36 @@ subroutine COP_ModelAdvance(gcomp, rc)
14231440
lbound(ptr3d,dim=3) /)
14241441
ub = (/ ubound(ptr3d,dim=1), ubound(ptr3d,dim=2), &
14251442
ubound(ptr3d,dim=3) /)
1443+
!
1444+
if (hasTop) ub(1) = ub(1)-1
1445+
if (hasRight) ub(2) = ub(2)-1
1446+
if (hasBottom) lb(1) = lb(1)+1
1447+
if (hasLeft) lb(2) = lb(2)+1
14261448
!
14271449
nPoints3D = 1
14281450
do k = 1, 3
14291451
nPoints3D = nPoints3D*(ub(k)-lb(k)+1)
14301452
end do
1453+
!
1454+
! write(*,fmt="(A,7I5,I10)") "field 3d = ", localPet, lb(1), ub(1), lb(2), ub(2), lb(3), ub(3), nPoints3D
14311455
!
14321456
if (.not. allocated(var1d)) then
14331457
allocate(var1d(nPoints3D))
14341458
end if
14351459
!
1436-
call ntooned_3d(lb, ub, ptr3d, var1d)
1460+
call ntooned_3d(localPet, .true., lb, ub, ptr3d(lb(1):ub(1),lb(2):ub(2),lb(3):ub(3)), var1d)
14371461
!
14381462
!-----------------------------------------------------------------------
14391463
! Add field to Catalyst
14401464
!-----------------------------------------------------------------------
14411465
!
14421466
gname = replace_str(gname, "_grid3d", "_input3d")
1467+
!
1468+
! write(*, fmt="(A,I5, 4D15.6)") "max/min = ", localPet, &
1469+
! minval(var1d), &
1470+
! maxval(var1d), &
1471+
! minval(minval(minval(ptr3d(lb(1):ub(1),lb(2):ub(2),lb(3):ub(3)), dim=1), dim=1)), &
1472+
! maxval(maxval(maxval(ptr3d(lb(1):ub(1),lb(2):ub(2),lb(3):ub(3)), dim=1), dim=1))
14431473
!
14441474
call add_scalar(var1d, trim(itemNameList(i))//char(0), nPoints3D, &
14451475
petCount, localPet, trim(gname)//char(0))
@@ -1475,6 +1505,27 @@ subroutine COP_ModelAdvance(gcomp, rc)
14751505
!
14761506
if (allocated(minIndexPTile)) deallocate(minIndexPTile)
14771507
if (allocated(maxIndexPTile)) deallocate(maxIndexPTile)
1508+
!
1509+
!-----------------------------------------------------------------------
1510+
! Debug: write out component grid in VTK format
1511+
!-----------------------------------------------------------------------
1512+
!
1513+
! if (debugLevel > 1) then
1514+
! call ESMF_GridWriteVTK(grid, filename="coproc_"//trim(gname), rc=rc)
1515+
! if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU,&
1516+
! line=__LINE__, file=FILENAME)) return
1517+
! end if
1518+
!
1519+
!-----------------------------------------------------------------------
1520+
! Debug: write field in netCDF format
1521+
!-----------------------------------------------------------------------
1522+
!
1523+
if (debugLevel == 3) then
1524+
write(ofile,fmt="(A)") 'cop_import_'//trim(itemNameList(i))//trim(str1)
1525+
call ESMF_FieldWrite(field, trim(ofile)//'.nc', rc=rc)
1526+
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
1527+
line=__LINE__, file=FILENAME)) return
1528+
end if
14781529
!
14791530
end do
14801531
!
@@ -1577,13 +1628,15 @@ subroutine ntooned_2d(lb, ub, xnd, x1d)
15771628
!
15781629
end subroutine ntooned_2d
15791630
!
1580-
subroutine ntooned_3d(lb, ub, xnd, x1d)
1631+
subroutine ntooned_3d(localPet, isprint, lb, ub, xnd, x1d)
15811632
implicit none
15821633
!
15831634
!-----------------------------------------------------------------------
15841635
! Imported variable declarations
15851636
!-----------------------------------------------------------------------
15861637
!
1638+
integer, intent(in) :: localPet
1639+
logical, intent(in) :: isprint
15871640
integer, intent(in) :: lb(3)
15881641
integer, intent(in) :: ub(3)
15891642
real*8 , intent(in) :: xnd(lb(1):ub(1),lb(2):ub(2),lb(3):ub(3))

‎util/mod_config.F90‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -597,13 +597,34 @@ subroutine read_config(vm, rc)
597597
end if
598598
!
599599
if (localPet == 0) then
600-
write(*, fmt='(A12,I2,A,I2)') "Co-processing Tiles: ", &
600+
write(*, fmt='(A,I2,A,I2)') "Co-processing Tiles: ", &
601601
models(Icopro)%tile(1),"x",models(Icopro)%tile(2)
602602
end if
603603
!
604604
end if
605605
!
606606
!-----------------------------------------------------------------------
607+
! Read width of halo or ghost region
608+
!-----------------------------------------------------------------------
609+
!
610+
if (models(Icopro)%modActive) then
611+
!
612+
call ESMF_ConfigFindLabel(cf, 'CoProcessorHaloWidth:', rc=rc)
613+
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
614+
line=__LINE__, file=FILENAME)) return
615+
!
616+
call ESMF_ConfigGetAttribute(cf, models(Icopro)%haloWidth, rc=rc)
617+
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
618+
line=__LINE__, file=FILENAME)) return
619+
!
620+
if (localPet == 0) then
621+
write(*, fmt='(A,I2)') "Co-processing Halo/Ghost Width: ", &
622+
models(Icopro)%haloWidth
623+
end if
624+
!
625+
end if
626+
!
627+
!-----------------------------------------------------------------------
607628
! Read vertical levels for model components
608629
! Only valid if co-processing component is activated
609630
!-----------------------------------------------------------------------

‎util/mod_types.F90‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,7 @@ module mod_types
114114
integer :: nPets
115115
logical :: modActive
116116
integer :: tile(2)
117+
integer :: haloWidth
117118
integer :: nLevs
118119
real(ESMF_KIND_R8), allocatable :: levs(:)
119120
integer, allocatable :: petList(:)

0 commit comments

Comments
 (0)