Flexible restart write times (restart_fh) for med, ocn, ice, wav

.gitmodules

@@ -16,20 +16,20 @@
   branch = develop
 [submodule "CMEPS"]
   path = CMEPS-interface/CMEPS
-  url = https://github.com/NOAA-EMC/CMEPS
-  branch = emc/develop
+  url = https://github.com/NickSzapiro-NOAA/CMEPS
+  branch = restart_fh
 [submodule "HYCOM"]
   path = HYCOM-interface/HYCOM
   url = https://github.com/NOAA-EMC/HYCOM-src
   branch = emc/develop
 [submodule "MOM6"]
   path = MOM6-interface/MOM6
-  url = https://github.com/NOAA-EMC/MOM6
-  branch = dev/emc
+  url = https://github.com/NickSzapiro-NOAA/MOM6
+  branch = restart_fh
 [submodule "CICE"]
   path = CICE-interface/CICE
-  url = https://github.com/NOAA-EMC/CICE
-  branch = develop
+  url = https://github.com/NickSzapiro-NOAA/CICE
+  branch = restart_fh
 [submodule "CDEPS"]
   path = CDEPS-interface/CDEPS
   url = https://github.com/NOAA-EMC/CDEPS

CDEPS-interface/cdeps_files.cmake

@@ -4,6 +4,7 @@ list(APPEND ufs_cdeps_share_files
   ufs/cdeps_share/shr_assert_mod.F90
   ufs/cdeps_share/shr_frz_mod.F90
   ufs/cdeps_share/shr_infnan_mod.F90
+  ufs/cdeps_share/shr_is_restart_fh_mod.F90
 )
 
 list(APPEND cdeps_share_files

CDEPS-interface/ufs/cdeps_share/shr_is_restart_fh_mod.F90

@@ -0,0 +1,146 @@
+module shr_is_restart_fh_mod
+
+  ! Common methods for components to check if it's time to write forecast hour-based restarts
+
+  !use dshr_methods_mod , only : chkerr
+  use ESMF, only : ESMF_ConfigCreate, ESMF_ConfigDestroy, ESMF_ConfigLoadFile, &
+                     ESMF_ConfigGetLen, ESMF_ConfigGetAttribute, ESMF_TimePrint, &
+                     ESMF_LOGMSG_INFO, ESMF_LogWrite, ESMF_TimeInterval, &
+                     ESMF_Time, ESMF_KIND_R8, ESMF_Config, ESMF_Clock, &
+                     ESMF_TimeIntervalSet, ESMF_TimePrint, operator(+), operator(==), &
+                     ESMF_LogFoundError, ESMF_LOGERR_PASSTHRU
+
+  implicit none
+  private
+  save
+
+  logical :: write_restartfh = .false.
+  type(ESMF_Time), allocatable :: restartFhTimes(:)
+!$OMP THREADPRIVATE (write_restartfh, restartFhTimes)
+
+  public :: init_is_restart_fh, is_restart_fh, finalize_restart_fh, write_restartfh
+
+contains
+
+  !-----------------------------------------------------------------------
+  subroutine init_is_restart_fh(currentTime, dtime, lLog)
+    !
+    ! !DESCRIPTION:
+    ! Process restart_fh attribute from model_configure in UFS
+    !
+    ! !USES:
+    !
+    ! !ARGUMENTS:
+    type(ESMF_Time), intent(in) :: currentTime
+    integer, intent(in)         :: dtime ! time step (s)
+    logical, intent(in)         :: lLog ! If true, this task logs restart_fh info
+    !
+    ! !LOCAL VARIABLES:
+    character(len=256)           :: timestr
+    integer                      :: n, nfh, fh_s, rc
+    logical                      :: isPresent
+    real(kind=ESMF_KIND_R8), allocatable :: restart_fh(:)
+    type(ESMF_TimeInterval)      :: fhInterval
+    type(ESMF_Config)            :: CF_mc
+
+    character(len=*), parameter :: subname = 'init_restart_fh'
+    !-----------------------------------------------------------------------
+
+    ! set up Times to write non-interval restarts
+    inquire(FILE='model_configure', EXIST=isPresent)
+    if (isPresent) then !model_configure exists. this is ufs run
+      CF_mc = ESMF_ConfigCreate(rc=rc)
+      call ESMF_ConfigLoadFile(config=CF_mc,filename='model_configure' ,rc=rc)
+      !if (ChkErr(rc,__LINE__,__FILE__)) return
+      if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return
+
+      nfh = ESMF_ConfigGetLen(config=CF_mc, label ='restart_fh:',rc=rc)
+      if (nfh .gt. 0) then
+        allocate(restart_fh(1:nfh))
+        allocate(restartFhTimes(1:nfh)) !not deallocated here
+
+        call ESMF_ConfigGetAttribute(CF_mc,valueList=restart_fh,label='restart_fh:', rc=rc)
+        !if (ChkErr(rc,__LINE__,__FILE__)) return
+        if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return
+        ! create a list of times at each restart_fh
+        do n = 1,nfh
+          fh_s = NINT(3600*restart_fh(n))
+          call ESMF_TimeIntervalSet(fhInterval, s=fh_s, rc=rc)
+          !if (ChkErr(rc,__LINE__,__FILE__)) return
+          if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return
+          restartFhTimes(n) = currentTime + fhInterval
+          call ESMF_TimePrint(restartFhTimes(n), options="string", &
+                              preString="restart_fh at ", unit=timestr, rc=rc)
+          !if (ChkErr(rc,__LINE__,__FILE__)) return
+          if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return
+          if (lLog) then
+            if (mod(fh_s,dtime) /= 0) then
+              call ESMF_LogWrite('restart time NOT to be written for '//trim(timestr), ESMF_LOGMSG_INFO)
+            else
+              call ESMF_LogWrite('restart time to be written for '//trim(timestr), ESMF_LOGMSG_INFO)
+            end if
+          end if
+        end do
+        deallocate(restart_fh)
+      end if !nfh>0
+      call ESMF_ConfigDestroy(CF_mc, rc=rc)
+      !if (ChkErr(rc,__LINE__,__FILE__)) return
+      if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return
+    end if !model_configure
+
+  end subroutine init_is_restart_fh
+
+  function is_restart_fh(clock) result(lWrite)
+    !
+    ! !DESCRIPTION:
+    ! True/false if time to write restart
+    !
+    ! !USES:
+    use ESMF, only : ESMF_ClockGetNextTime
+
+    !
+    ! !ARGUMENTS:
+    type(ESMF_Clock), intent(in) :: clock
+    logical :: lWrite ! function result
+    !
+    ! !LOCAL VARIABLES:
+    integer                    :: nfh, rc
+    type(ESMF_Time)            :: nextTime
+
+    character(len=*), parameter :: subname = 'is_restart_fh'
+    !-----------------------------------------------------------------------
+
+    write_restartfh = .false.
+    if (allocated(restartFhTimes)) then
+      ! check if next time is == to any restartfhtime
+      do nfh = 1,size(restartFhTimes)
+        call ESMF_ClockGetNextTime(clock, nextTime=nexttime, rc=rc)
+        !if (ChkErr(rc,__LINE__,__FILE__)) return
+        if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return
+        if (nextTime == restartFhTimes(nfh)) write_restartfh = .true.
+      end do
+    end if
+
+    lWrite = write_restartfh
+
+  end function is_restart_fh
+
+  subroutine finalize_restart_fh()
+    !
+    ! !DESCRIPTION:
+    ! Clean-up...release allocated memory
+    !
+    ! !USES:
+    !
+    ! !ARGUMENTS:
+    !
+    ! !LOCAL VARIABLES:
+
+    character(len=*), parameter :: subname = 'finalize_restart_fh'
+    !-----------------------------------------------------------------------
+
+    if (allocated(restartFhTimes)) deallocate(restartFhTimes)
+
+  end subroutine finalize_restart_fh
+
+end module shr_is_restart_fh_mod

CICE-interface/cice_files.cmake

@@ -151,8 +151,8 @@ list(APPEND cice_nuopc_cmeps_driver_files
   CICE/cicecore/drivers/nuopc/cmeps/ice_mesh_mod.F90
 )
 
-#-- Using ice prescribed ifndef cesmcoupled
 list(APPEND cice_cdeps_inline_files
+  #-- Using ice prescribed ifndef cesmcoupled
   ${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/share/shr_orb_mod.F90
   ${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/share/shr_const_mod.F90
   ${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/share/shr_abort_mod.F90
@@ -168,4 +168,7 @@ list(APPEND cice_cdeps_inline_files
   ${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/streams/dshr_stream_mod.F90
   ${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/streams/dshr_methods_mod.F90
   ${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/dshr/dshr_mod.F90
+
+  #restart_fh
+  ${PROJECT_SOURCE_DIR}/CDEPS-interface/ufs/cdeps_share/shr_is_restart_fh_mod.F90
 )

CMEPS-interface/CMakeLists.txt

@@ -42,7 +42,9 @@ list(APPEND _ufs_util_files
   ${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/share/shr_sys_mod.F90
   ${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/share/shr_kind_mod.F90
   ${PROJECT_SOURCE_DIR}/CDEPS-interface/ufs/cdeps_share/shr_assert_mod.F90
-  ${PROJECT_SOURCE_DIR}/CDEPS-interface/ufs/cdeps_share/shr_infnan_mod.F90)
+  ${PROJECT_SOURCE_DIR}/CDEPS-interface/ufs/cdeps_share/shr_infnan_mod.F90
+  ${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/streams/dshr_methods_mod.F90
+  ${PROJECT_SOURCE_DIR}/CDEPS-interface/ufs/cdeps_share/shr_is_restart_fh_mod.F90)
 
 list(APPEND _mediator_files
   CMEPS/mediator/med_phases_restart_mod.F90
@@ -190,10 +192,10 @@ if(OpenMP_Fortran_FOUND)
 endif()
 
 ### Set dependency for CDEPS if inline capability is requested
-if(CDEPS_INLINE)
+#if(CDEPS_INLINE)
   add_dependencies(cmeps cdeps::cdeps)
   target_link_libraries(cmeps PUBLIC cdeps::cdeps)
-endif()
+#endif()
 
 ###############################################################################
 ### Install

MOM6-interface/CMakeLists.txt

@@ -108,6 +108,10 @@ if(MOM6SOLO)
   #endif()
 endif()
 
+add_dependencies(mom6 cdeps::cdeps)
+target_compile_definitions(mom6 PUBLIC "DISABLE_FoX")
+target_link_libraries(mom6 PUBLIC cdeps::cdeps)
+
 ###############################################################################
 ### Install
 ###############################################################################

MOM6-interface/mom6_files.cmake

@@ -315,6 +315,10 @@ list(APPEND mom6_nuopc_src_files
   MOM6/config_src/drivers/timing_tests/time_MOM_EOS.F90
 )
 
+list(APPEND mom6_nuopc_src_files
+  ${PROJECT_SOURCE_DIR}/CDEPS-interface/ufs/cdeps_share/shr_is_restart_fh_mod.F90
+)
+
 list(APPEND mom6_solo_src_files
   MOM6/config_src/drivers/solo_driver/MESO_surface_forcing.F90
   MOM6/config_src/drivers/solo_driver/MOM_driver.F90

tests/default_vars.sh

@@ -1199,6 +1199,7 @@ export_cmeps() {
   export pio_rearranger=box
   export RUNTYPE=startup
   export RESTART_N=${FHMAX}
+  export RESTART_FH="-999"
   export CMEPS_RESTART_DIR=./RESTART/
   export cap_dbug_flag=0
   export WRITE_ENDOFRUN_RESTART=.false.

tests/parm/datm_cdeps_configure.IN

@@ -7,3 +7,4 @@ start_second:            0
 nhours_fcst:             @[FHMAX]
 dt_atmos:                @[DT_ATMOS]
 fhrot:                   @[FHROT]
+restart_fh:              @[RESTART_FH]

tests/tests/datm_cdeps_gfs

@@ -6,7 +6,10 @@ export TEST_DESCR="DATM_CDEPS-MOM6-CICE-CMEPS_GFS - control "
 
 export CNTL_DIR=datm_cdeps_gfs
 
-export LIST_FILES="RESTART/20210323.060000.MOM.res.nc \
+export LIST_FILES="RESTART/iced.2021-03-22-23400.nc \
+                   RESTART/DATM_GFS.cpl.r.2021-03-22-64800.nc \
+                   RESTART/20210322.090000.MOM.res.nc \
+                   RESTART/20210323.060000.MOM.res.nc \
                    RESTART/iced.2021-03-23-21600.nc \
                    RESTART/DATM_GFS.cpl.r.2021-03-23-21600.nc"
 
@@ -28,6 +31,7 @@ export atm_datamode=GEFS
 export CICE_KTHERM=2
 export CICE_TFREEZE_OPTION=mushy
 export RESTART_N=12
+export RESTART_FH="0.5 3"
 export stream_files="INPUT/gfs.202103.nc"
 export eps_imesh=2.5e-1
 export MOM6_TOPOEDITS=ufs.topo_edits_011818.nc