Merge pull request #1176 from mwaxmonsky/vertical-diffusion-refactor

cam6_4_045: Vertical diffusion refactor

Author: mwaxmonsky

.gitmodules

@@ -36,7 +36,7 @@
 [submodule "atmos_phys"]
 	path = src/atmos_phys
 	url = https://github.com/ESCOMP/atmospheric_physics
-        fxtag = atmos_phys0_05_001
+        fxtag = atmos_phys0_06_000
         fxrequired = AlwaysRequired
 	fxDONOTUSEurl = https://github.com/ESCOMP/atmospheric_physics
 

bld/configure

@@ -2335,6 +2335,7 @@ sub write_filepath
     # be overridden by modules from directories that occur earlier
     # in the list of filepaths.
     print $fh "$camsrcdir/src/physics/cam\n";
+    print $fh "$camsrcdir/src/atmos_phys/to_be_ccppized\n";
 
     #Add the CCPP'ized subdirectories
     print $fh "$camsrcdir/src/atmos_phys/schemes/tropopause_find\n";

cime_config/config_pes.xml

@@ -164,14 +164,14 @@
 	  <ntasks_cpl>-4</ntasks_cpl>
 	</ntasks>
 	<nthrds>
-	  <nthrds_atm>2</nthrds_atm>
-	  <nthrds_lnd>2</nthrds_lnd>
-	  <nthrds_rof>2</nthrds_rof>
-	  <nthrds_ice>2</nthrds_ice>
-	  <nthrds_ocn>2</nthrds_ocn>
-	  <nthrds_glc>2</nthrds_glc>
-	  <nthrds_wav>2</nthrds_wav>
-	  <nthrds_cpl>2</nthrds_cpl>
+	  <nthrds_atm>1</nthrds_atm>
+	  <nthrds_lnd>1</nthrds_lnd>
+	  <nthrds_rof>1</nthrds_rof>
+	  <nthrds_ice>1</nthrds_ice>
+	  <nthrds_ocn>1</nthrds_ocn>
+	  <nthrds_glc>1</nthrds_glc>
+	  <nthrds_wav>1</nthrds_wav>
+	  <nthrds_cpl>1</nthrds_cpl>
 	</nthrds>
 	<rootpe>
 	  <rootpe_atm>0</rootpe_atm>

doc/ChangeLog

@@ -1,5 +1,98 @@
 ===============================================================
 
+Tag name: cam6_4_045
+Originator(s): mwaxmonsky 
+Date: 10/25/2024
+One-line Summary: Start refactoring of vertical diffusion to be CCPPized
+Github PR URL: https://github.com/ESCOMP/CAM/pull/1176
+
+Purpose of changes (include the issue number and title text for each relevant GitHub issue):
+Start refactoring of vertical diffusion to be CCPPized and outsources solve phase of non-graft decomp solves to atmospheric physics solver.
+
+Describe any changes made to build system:
+- Adds the to_be_ccppized directory in atmospheric_physics to source search path
+
+Describe any changes made to the namelist: N/A
+
+List any changes to the defaults for the boundary datasets: N/A
+
+Describe any substantial timing or memory changes: N/A
+
+Code reviewed by:
+
+List all files eliminated:
+D       src/utils/coords_1d.F90
+D       src/utils/linear_1d_operators.F90
+- Files were moved to atmospheric_physics
+
+List all files added and what they do: N/A
+
+List all existing files that have been modified, and describe the changes:
+M       .gitmodules
+M       bld/configure
+M       src/atmos_phys
+- Update atmospheric_physics reference and added new atmospheric_physics directory to search path
+
+M       cime_config/config_pes.xml
+- Decreasing NTHRDS from 2 to 1 to force single threaded execution as multithreaded execution enables the -smp flag which breaks under OpenMP when running the SE dycore, similar to #1087
+
+M       src/physics/cam/diffusion_solver.F90
+- Uses new CCPPized interface for decomp that was moved to atmospheric_physics repo to single solve step.
+
+If there were any failures reported from running test_driver.sh on any test
+platform, and checkin with these failures has been OK'd by the gatekeeper,
+then copy the lines from the td.*.status files for the failed tests to the
+appropriate machine below.  All failed tests must be justified.
+
+derecho/intel/aux_cam:
+
+ERP_Ln9.f09_f09_mg17.FCSD_HCO.derecho_intel.cam-outfrq9s (Overall: FAIL) details:
+SMS_Ld1.f09_f09_mg17.FCHIST_GC.derecho_intel.cam-outfrq1d (Overall: FAIL) details:
+- pre-existing failure due to HEMCO not having reproducible results issues #1018 and #856
+
+SMS_D_Ln9.f19_f19_mg17.FXHIST.derecho_intel.cam-outfrq9s_amie (Overall: FAIL)
+SMS_D_Ln9_P1280x1.ne0CONUSne30x8_ne0CONUSne30x8_mt12.FCHIST.derecho_intel.cam-outfrq9s (Overall: FAIL)
+- pre-existing failures due to build-namelist error requiring CLM/CTSM external update.
+
+derecho/nvhpc/aux_cam: All PASS
+
+izumi/nag/aux_cam:
+
+DAE.f45_f45_mg37.FHS94.izumi_nag.cam-dae (Overall: FAIL) details:
+ - pre-existing failure -- issue #670
+
+izumi/gnu/aux_cam: All PASS
+
+CAM tag used for the baseline comparison tests if different than previous
+tag:
+
+Summarize any changes to answers, i.e.,
+- what code configurations:
+- what platforms/compilers:
+- nature of change (roundoff; larger than roundoff but same climate; new
+  climate):
+N/A
+
+If bitwise differences were observed, how did you show they were no worse
+than roundoff?
+N/A
+
+If this tag changes climate describe the run(s) done to evaluate the new
+climate in enough detail that it(they) could be reproduced, i.e.,
+- source tag (all code used must be in the repository):
+- platform/compilers:
+- configure commandline:
+- build-namelist command (or complete namelist):
+- MSS location of output:
+N/A
+
+MSS location of control simulations used to validate new climate: N/A
+
+URL for AMWG diagnostics output used to validate new climate: N/A
+
+===============================================================
+===============================================================
+
 Tag name: cam6_4_044
 Originator(s): eaton
 Date: 04 November 2024

src/atmos_phys

@@ -161,6 +161,7 @@ subroutine compute_vdiff( lchnk           ,
     use linear_1d_operators, only : BoundaryType, BoundaryFixedLayer, &
          BoundaryData, BoundaryFlux, TriDiagDecomp
     use vdiff_lu_solver,     only : fin_vol_lu_decomp
+    use vertical_diffusion_solver, only : fin_vol_solve
     use beljaars_drag_cam,   only : do_beljaars
     ! FIXME: This should not be needed
     use air_composition,     only: rairv
@@ -570,12 +571,15 @@ end function vd_lu_qdecomp
           tau_damp_rate(:,k) = tau_damp_rate(:,k) + dragblj(:ncol,k)
        end do
 
-       decomp = fin_vol_lu_decomp(ztodt, p, &
-            coef_q=tau_damp_rate, coef_q_diff=kvm(:ncol,:)*dpidz_sq)
+       v(:ncol,:) = fin_vol_solve(ztodt, p, v(:ncol,:), ncol, pver, &
+                         coef_q=tau_damp_rate,                      &
+                         coef_q_diff=kvm(:ncol,:)*dpidz_sq)
+
+       u(:ncol,:) = fin_vol_solve(ztodt, p, u(:ncol,:), ncol, pver, &
+                         coef_q=tau_damp_rate,                      &
+                         coef_q_diff=kvm(:ncol,:)*dpidz_sq)
+
 
-       call decomp%left_div(u(:ncol,:))
-       call decomp%left_div(v(:ncol,:))
-       call decomp%finalize()
 
        ! ---------------------------------------------------------------------- !
        ! Calculate 'total' ( tautotx ) and 'tms' ( tautmsx ) stresses that      !
@@ -741,37 +745,31 @@ end function vd_lu_qdecomp
 
           ! Boundary layer thickness of "0._r8" signifies that the boundary
           ! condition is defined directly on the top interface.
-          decomp = fin_vol_lu_decomp(ztodt, p, &
-               coef_q_diff=kvh(:ncol,:)*dpidz_sq, &
-               upper_bndry=interface_boundary)
 
           if (.not. use_spcam) then
-           call decomp%left_div(dse(:ncol,:), &
-                l_cond=BoundaryData(dse_top(:ncol)))
+             dse(:ncol,:) = fin_vol_solve(ztodt, p, dse(:ncol,:), ncol, pver, &
+                                  coef_q_diff=kvh(:ncol,:)*dpidz_sq,          &
+                                  upper_bndry=interface_boundary,             &
+                                  l_cond=BoundaryData(dse_top(:ncol)))
           endif
 
-          call decomp%finalize()
-
           ! Calculate flux at top interface
 
           ! Modification : Why molecular diffusion does not work for dry static energy in all layers ?
 
           topflx(:ncol) =  - kvh(:ncol,1) * tmpi2(:ncol,1) / (ztodt*gravit) * &
                ( dse(:ncol,1) - dse_top(:ncol) )
 
-          decomp = fin_vol_lu_decomp(ztodt, p, &
-               coef_q_diff=kvt(:ncol,:)*dpidz_sq, &
-               coef_q_weight=cpairv(:ncol,:))
-
           ttemp0 = t(:ncol,:)
           ttemp = ttemp0
 
           ! upper boundary is zero flux for extended model
           if (.not. use_spcam) then
-             call decomp%left_div(ttemp)
+             ttemp = fin_vol_solve(ztodt, p, ttemp, ncol, pver, &
+                          coef_q_diff=kvt(:ncol,:)*dpidz_sq,    &
+                          coef_q_weight=cpairv(:ncol,:))
           end if
 
-          call decomp%finalize()
 
           !-------------------------------------
           !  Update dry static energy
@@ -791,17 +789,13 @@ end function vd_lu_qdecomp
 
           ! Boundary layer thickness of "0._r8" signifies that the boundary
           ! condition is defined directly on the top interface.
-          decomp = fin_vol_lu_decomp(ztodt, p, &
-               coef_q_diff=kv_total(:ncol,:)*dpidz_sq, &
-               upper_bndry=interface_boundary)
-
           if (.not. use_spcam) then
-             call decomp%left_div(dse(:ncol,:), &
-                  l_cond=BoundaryData(dse_top(:ncol)))
+             dse(:ncol,:) = fin_vol_solve(ztodt, p, dse(:ncol,:), ncol, pver, &
+                                 coef_q_diff=kv_total(:ncol,:)*dpidz_sq,      &
+                                 upper_bndry=interface_boundary,              &
+                                 l_cond=BoundaryData(dse_top(:ncol)))
           end if
 
-          call decomp%finalize()
-
           ! Calculate flux at top interface
 
           ! Modification : Why molecular diffusion does not work for dry static energy in all layers ?