[production/RRFS.v1] fix RRFS/REFS restart reproducibility and DEBUG crash issues for RRFSv1 operational implementation (#2925)

JiliDong-NOAA · web-flow · commit 8b73ad3d81aa · 2026-01-20T14:48:46.000-05:00
* Update .gitmodules * Fix rrfs (conus13km) decomp and restart tests * update .gitmodules * Update FV3 * update for rrfsens restart bitwise reproducibility * update FV3 * revert FV3 * update FV3 * REFS ensemble restart fix (#43) * update for rrfsens restart bitwise reproducibility --------- * update FV3 and stochastic_physics; fix DEBUG crash for REFS * update FV3 * update FV3 to fix C3 compiling error * bl_date update * WCOSS2 RRFS Tests Pass * add hercules RT log: passed * remove unused conf file * revert ufsatm and stochastic_physics back to community * update ufsatm and stochastic_physics ---------
diff --git a/FV3 b/FV3
@@ -1 +1 @@
-Subproject commit 81e6d10dea838f03aeffdd70e8e9c9ce7001304d
+Subproject commit f861e49ebf73c26535848615dbb7f9718076264f
diff --git a/stochastic_physics b/stochastic_physics
@@ -1 +1 @@
-Subproject commit a5561802021d89a9a1e2b52cb69393efcdc6f71c
+Subproject commit e8d56ddc200208560b21abef279462c9ef276ecc
diff --git a/tests/bl_date.conf b/tests/bl_date.conf
@@ -1 +1 @@
-export BL_DATE=20250123
+export BL_DATE=20251017
diff --git a/tests/default_vars.sh b/tests/default_vars.sh
@@ -1402,8 +1402,8 @@ export SHAL_CNV=.false.
 export DO_SAT_ADJ=.false.
 export DO_DEEP=.false.
 export CCPP_SUITE='FV3_HRRR'
-export INPES=12
-export JNPES=12
+export INPES=10
+export JNPES=8
 export NPX=397
 export NPY=233
 export NPZ=65
diff --git a/tests/fv3_conf/fv3_qsub.IN_acorn b/tests/fv3_conf/fv3_qsub.IN_acorn
@@ -5,15 +5,16 @@
 #PBS -N @[JBNME]
 #PBS -A @[ACCNR]
 #PBS -q @[QUEUE]
-#PBS -l place=vscatter,select=@[NODES]:ncpus=@[TPN]:mpiprocs=@[TPN]:mem=500G
+#PBS -l place=vscatter,select=@[NODES]:ncpus=@[NCPUS]:mpiprocs=@[TPN]:ompthreads=@[THRD]:mem=500G
 #PBS -l place=excl
 #PBS -l walltime=00:@[WLCLK]:00
 
 set -eux
-echo -n " $( date +%s )," >  job_timestamp.txt
 
 cd $PBS_O_WORKDIR
 
+echo -n " $( date +%s )," >  job_timestamp.txt
+
 set +x
 module use $PWD/modulefiles
 module load modules.fv3
@@ -30,7 +31,7 @@ export ESMF_RUNTIME_COMPLIANCECHECK=OFF:depth=4
 export ESMF_RUNTIME_PROFILE=ON
 export ESMF_RUNTIME_PROFILE_OUTPUT="SUMMARY"
 
-mpiexec -n @[TASKS] -ppn @[TPN] -depth @[THRD] ./fv3.exe
+mpiexec -n @[TASKS] -ppn @[TPN] --cpu-bind core -depth @[THRD] ./fv3.exe
 
 echo "Model ended:    " `date`
 echo -n " $( date +%s )," >> job_timestamp.txt
diff --git a/tests/fv3_conf/fv3_qsub.IN_wcoss2 b/tests/fv3_conf/fv3_qsub.IN_wcoss2
@@ -5,15 +5,16 @@
 #PBS -N @[JBNME]
 #PBS -A @[ACCNR]
 #PBS -q @[QUEUE]
-#PBS -l place=vscatter,select=@[NODES]:ncpus=@[TPN]:mpiprocs=@[TPN]:mem=500G
+#PBS -l place=vscatter,select=@[NODES]:ncpus=@[NCPUS]:mpiprocs=@[TPN]:ompthreads=@[THRD]:mem=500G
 #PBS -l place=excl
 #PBS -l walltime=00:@[WLCLK]:00
 
 set -eux
-echo -n " $( date +%s )," >  job_timestamp.txt
 
 cd $PBS_O_WORKDIR
 
+echo -n " $( date +%s )," >  job_timestamp.txt
+
 set +x
 module use $PWD/modulefiles
 module load modules.fv3
@@ -30,7 +31,7 @@ export ESMF_RUNTIME_COMPLIANCECHECK=OFF:depth=4
 export ESMF_RUNTIME_PROFILE=ON
 export ESMF_RUNTIME_PROFILE_OUTPUT="SUMMARY"
 
-mpiexec -n @[TASKS] -ppn @[TPN] -depth @[THRD] ./fv3.exe
+mpiexec -n @[TASKS] -ppn @[TPN] --cpu-bind core -depth @[THRD] ./fv3.exe
 
 echo "Model ended:    " `date`
 echo -n " $( date +%s )," >> job_timestamp.txt
diff --git a/tests/logs/RegressionTests_hercules.log b/tests/logs/RegressionTests_hercules.log
diff --git a/tests/logs/RegressionTests_wcoss2.log b/tests/logs/RegressionTests_wcoss2.log
diff --git a/tests/rt.conf b/tests/rt.conf
@@ -199,12 +199,9 @@ RUN | hrrr_control_restart_dyn32_phy32                  | - noaacloud
 
 COMPILE | rrfs_dyn32_phy32_faster | intel | -DAPP=ATM -DFASTER=ON -DCCPP_SUITES=FV3_HRRR -D32BIT=ON -DCCPP_32BIT=ON | - noaacloud  | fv3 |
 RUN | conus13km_control             | - noaacloud                                      | baseline |
-RUN | conus13km_2threads            | - noaacloud                                      |          | conus13km_control
-RUN | conus13km_restart_mismatch    | - noaacloud                                      | baseline | conus13km_control
-
-# Expected to fail:
-# RUN | conus13km_restart           | - noaacloud                                      |          | conus13km_control
-# RUN | conus13km_decomp            | - noaacloud                                      |          | conus13km_control
+RUN | conus13km_2threads            | - noaacloud                                      |          |
+RUN | conus13km_restart             | - noaacloud                                      |          | conus13km_control
+RUN | conus13km_decomp              | - noaacloud                                      |          |
 
 COMPILE | rrfs_dyn64_phy32 | intel | -DAPP=ATM -DCCPP_SUITES=FV3_RAP,FV3_HRRR -DCCPP_32BIT=ON | - noaacloud  | fv3 |
 RUN | rap_control_dyn64_phy32                           |  - noaacloud                 | baseline |
@@ -213,13 +210,10 @@ COMPILE | rrfs_dyn32_phy32_debug | intel | -DAPP=ATM -DCCPP_SUITES=FV3_RAP,FV3_H
 RUN | rap_control_debug_dyn32_phy32                     |  - noaacloud                 | baseline |
 RUN | hrrr_control_debug_dyn32_phy32                    |  - noaacloud                 | baseline |
 RUN | conus13km_debug                                   |  - noaacloud                 | baseline |
-RUN | conus13km_debug_qr                                |  - derecho noaacloud         |          |
-RUN | conus13km_debug_2threads                          |  - derecho noaacloud         |          |
+RUN | conus13km_debug_qr                                |  - noaacloud                 |          |
+RUN | conus13km_debug_2threads                          |  - noaacloud                 |          |
 RUN | conus13km_radar_tten_debug                        |  - noaacloud                 | baseline |
 
-# Expected to fail:
-# RUN | conus13km_debug_decomp                          |  - noaacloud                 |          |
-
 COMPILE | rrfs_dyn64_phy32_debug | intel | -DAPP=ATM -DCCPP_SUITES=FV3_RAP,FV3_HRRR -DCCPP_32BIT=ON -DDEBUG=ON | - noaacloud  | fv3 |
 RUN | rap_control_dyn64_phy32_debug                     |  - noaacloud                 | baseline |
 
@@ -371,12 +365,9 @@ RUN | hrrr_control_decomp_dyn32_phy32                   | + hera hercules
 RUN | rap_restart_dyn32_phy32                           | + hera hercules            |          | rap_control_dyn32_phy32
 RUN | hrrr_control_restart_dyn32_phy32                  | + hera hercules            |          | hrrr_control_dyn32_phy32
 RUN | conus13km_control                                 | + hera hercules            | baseline |
-RUN | conus13km_2threads                                | + hera hercules            |          | conus13km_control
-RUN | conus13km_restart_mismatch                        | + hera hercules            | baseline | conus13km_control
-
-# Expected to fail:
-# RUN | conus13km_restart                               | + hera hercules            |          | conus13km_control
-# RUN | conus13km_decomp                                | + hera hercules            |          | conus13km_control
+RUN | conus13km_2threads                                | + hera hercules            |          |
+RUN | conus13km_decomp                                  | + hera hercules            |          |
+RUN | conus13km_restart                                 | + hera hercules            |          | conus13km_control
 
 COMPILE | atm_dyn64_phy32 | gnu | -DAPP=ATM -DCCPP_32BIT=ON | + hera hercules | fv3 |
 RUN | rap_control_dyn64_phy32                           | + hera hercules            | baseline |
@@ -387,11 +378,9 @@ RUN | hrrr_control_debug_dyn32_phy32                    | + hera hercules
 RUN | conus13km_debug                                   | + hera hercules            | baseline |
 RUN | conus13km_debug_qr                                | + hera hercules            |          |
 RUN | conus13km_debug_2threads                          | + hera hercules            |          |
+RUN | conus13km_debug_decomp                            | + hera hercules            |          |
 RUN | conus13km_radar_tten_debug                        | + hera hercules            | baseline |
 
-# Expected to fail:
-# RUN | conus13km_debug_decomp       | + hera derecho hercules            |          |
-
 COMPILE | atm_dyn64_phy32_debug | gnu | -DAPP=ATM -DCCPP_32BIT=ON -DDEBUG=ON | + hera hercules | fv3 |
 RUN | rap_control_dyn64_phy32_debug                     | + hera hercules            | baseline |
 
diff --git a/tests/rt.conf_rrfs b/tests/rt.conf_rrfs
@@ -86,12 +86,9 @@ RUN | hrrr_control_restart_dyn32_phy32                  | - noaacloud
 
 COMPILE | rrfs_dyn32_phy32_faster | intel | -DAPP=ATM -DFASTER=ON -DCCPP_SUITES=FV3_HRRR -D32BIT=ON -DCCPP_32BIT=ON -DENABLE_PARALLELRESTART=NO -DENABLE_RRFS_WAR=NO | - noaacloud  | fv3 |
 RUN | conus13km_control             | - noaacloud                                      | baseline |
-RUN | conus13km_2threads            | - noaacloud                                      |          | conus13km_control
-RUN | conus13km_restart_mismatch    | - noaacloud                                      | baseline | conus13km_control
-
-# Expected to fail:
-# RUN | conus13km_restart           | - noaacloud                                      |          | conus13km_control
-# RUN | conus13km_decomp            | - noaacloud                                      |          | conus13km_control
+RUN | conus13km_2threads            | - noaacloud                                      |          |
+RUN | conus13km_decomp              | - noaacloud                                      |          |
+RUN | conus13km_restart             | - noaacloud                                      |          | conus13km_control
 
 COMPILE | rrfs_dyn64_phy32 | intel | -DAPP=ATM -DCCPP_SUITES=FV3_RAP,FV3_HRRR -DCCPP_32BIT=ON -DENABLE_PARALLELRESTART=NO -DENABLE_RRFS_WAR=NO | - noaacloud  | fv3 |
 RUN | rap_control_dyn64_phy32                           |  - noaacloud                 | baseline |
@@ -102,11 +99,9 @@ RUN | hrrr_control_debug_dyn32_phy32                    |  - noaacloud
 RUN | conus13km_debug                                   |  - noaacloud                 | baseline |
 RUN | conus13km_debug_qr                                |  - derecho noaacloud         |          |
 RUN | conus13km_debug_2threads                          |  - derecho noaacloud         |          |
+RUN | conus13km_debug_decomp                            |  - noaacloud                 |          |
 RUN | conus13km_radar_tten_debug                        |  - noaacloud                 | baseline |
 
-# Expected to fail:
-# RUN | conus13km_debug_decomp                          |  - noaacloud                 |          |
-
 COMPILE | rrfs_dyn64_phy32_debug | intel | -DAPP=ATM -DCCPP_SUITES=FV3_RAP,FV3_HRRR -DCCPP_32BIT=ON -DDEBUG=ON -DENABLE_PARALLELRESTART=NO -DENABLE_RRFS_WAR=NO | - noaacloud  | fv3 |
 RUN | rap_control_dyn64_phy32_debug                     |  - noaacloud                 | baseline |
 
diff --git a/tests/run_test.sh b/tests/run_test.sh
@@ -312,6 +312,8 @@ fi
 export PPN
 export UFS_TASKS
 
+export NCPUS=$(( TPN * THRD ))
+
 if [[ $SCHEDULER = 'pbs' ]]; then
   if [[ -e $PATHRT/fv3_conf/fv3_qsub.IN_${MACHINE_ID} ]]; then
     atparse < $PATHRT/fv3_conf/fv3_qsub.IN_${MACHINE_ID} > job_card
diff --git a/tests/tests/conus13km_2threads b/tests/tests/conus13km_2threads
@@ -17,11 +17,12 @@ export LIST_FILES="sfcf000.nc \
                    atmf001.nc"
 
 export_hrrr_conus13km
-export SMONTH=5
 export RESTART_INTERVAL=1
 export FHMAX=1
 export atm_omp_num_threads=2
-export INPES=12
-export JNPES=12
-export WRTTASK_PER_GROUP=6
 export QUILTING_RESTART=.false.
+export USE_MERRA2=.true.
+
+if [[ " hera orion hercules jet " =~ " ${MACHINE_ID} " ]] ; then
+  ZSTANDARD_LEVEL=5
+fi
diff --git a/tests/tests/conus13km_debug_2threads b/tests/tests/conus13km_debug_2threads
@@ -20,7 +20,4 @@ export_hrrr_conus13km
 export RESTART_INTERVAL=1
 export FHMAX=1
 export atm_omp_num_threads=2
-export INPES=12
-export JNPES=12
-export WRTTASK_PER_GROUP=6
 export QUILTING_RESTART=.false.
diff --git a/tests/tests/conus13km_debug_decomp b/tests/tests/conus13km_debug_decomp
@@ -19,6 +19,6 @@ export LIST_FILES="sfcf000.nc \
 export_hrrr_conus13km
 export RESTART_INTERVAL=1
 export FHMAX=1
-export INPES=16
-export JNPES=9
+export INPES=8
+export JNPES=10
 export QUILTING_RESTART=.false.
diff --git a/tests/tests/conus13km_debug_qr b/tests/tests/conus13km_debug_qr
@@ -1,13 +1,13 @@
 ###############################################################################
 #
-#  HRRR physics on 13km domain, control
+#  HRRR physics on 13km domain, debug with quilting restart
 #
 ###############################################################################
 
 # This configuration is supposed to match the rrfs_dev1 parallel, so
 # do not change the namelist without checking the current parallel.
 
-export TEST_DESCR="HRRR physics on 13km domain, control"
+export TEST_DESCR="HRRR physics on 13km domain, debug with quilting restart"
 
 export CNTL_DIR=conus13km_debug
 
@@ -26,3 +26,4 @@ export LIST_FILES="sfcf000.nc \
 export_hrrr_conus13km
 export RESTART_INTERVAL=1
 export FHMAX=1
+export QUILTING_RESTART=.true.
diff --git a/tests/tests/conus13km_decomp b/tests/tests/conus13km_decomp
@@ -20,6 +20,11 @@ export LIST_FILES="sfcf000.nc \
 
 export_hrrr_conus13km
 export RESTART_INTERVAL=1
-export INPES=16
-export JNPES=9
+export INPES=8
+export JNPES=10
 export QUILTING_RESTART=.false.
+export USE_MERRA2=.true.
+
+if [[ " hera orion hercules jet " =~ " ${MACHINE_ID} " ]] ; then
+  ZSTANDARD_LEVEL=5
+fi
diff --git a/tests/tests/conus13km_restart b/tests/tests/conus13km_restart
@@ -19,3 +19,8 @@ export FHROT=1
 export RESTART_FILE_PREFIX="${SYEAR}${SMONTH}${SDAY}.$(printf "%02d" $(( ${SHOUR} + ${FHROT}  )))0000"
 export RRFS_RESTART=YES
 export QUILTING_RESTART=.false.
+export USE_MERRA2=.true.
+
+if [[ " hera orion hercules jet " =~ " ${MACHINE_ID} " ]] ; then
+  ZSTANDARD_LEVEL=5
+fi

Original file line number	Diff line number	Diff line change
`@@ -1 +1 @@`
`1`		`-export BL_DATE=20250123`
	`1`	`+export BL_DATE=20251017`