Merge branch 'bartgol/eamxx/step-tends-refactor' (PR #7824)

Allow calculation also of tendency of groups monolithic/individual fields.

[non-BFB] Only for newly added var to eamxx-prod output.

Author: bartgol

components/eamxx/cime_config/namelist_defaults_eamxx.xml

@@ -153,7 +153,7 @@ be lost if SCREAM_HACK_XML is not enabled.
       >0</internal_diagnostics_level>
       <compute_tendencies
         type="array(string)"
-        doc="list of computed fields for which this process will back out tendencies"
+        doc="list of computed fields/groups for which this process will back out tendencies"
         />
     </atm_proc_base>
 

components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/shell_commands

@@ -25,7 +25,7 @@ $atmchange -b physics::cosp::cosp_frequency=1
 $atmchange -b physics::mac_aero_mic::shoc::compute_tendencies=T_mid,qv
 $atmchange -b physics::mac_aero_mic::p3::compute_tendencies=T_mid,qv
 $atmchange -b physics::rrtmgp::compute_tendencies=T_mid
-$atmchange -b homme::compute_tendencies=T_mid,qv
+$atmchange -b homme::compute_tendencies=T_mid,qv,qc
 
 # Set temperature cut off in dycore threshold to 180K
 $atmchange -b vtheta_thresh=180

components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.4hourlyAVG_coarse.yaml

@@ -16,6 +16,7 @@ fields:
     - p3_qv_tend
     - shoc_qv_tend
     - homme_qv_tend
+    - homme_qc_tend
     - SW_flux_dn
     - SW_flux_up
     - LW_flux_dn

components/eamxx/src/control/atmosphere_driver.cpp

@@ -310,10 +310,6 @@ void AtmosphereDriver::create_grids()
   // Each process will grab what they need
   m_atm_process_group->set_grids(m_grids_manager);
 
-
-  // Also make each atm proc build requests for tendency fields, if needed
-  m_atm_process_group->setup_tendencies_requests();
-
   m_ad_status |= s_grids_created;
 
   stop_timer("EAMxx::create_grids");
@@ -592,6 +588,9 @@ void AtmosphereDriver::create_fields()
     m_atm_process_group->set_required_field(m_field_mgr->get_field(fid).get_const());
   }
 
+  // Make atm procs create the proc-level tendency fields (if requested)
+  m_atm_process_group->setup_step_tendencies(m_grids_manager->get_grid("physics")->name());
+
   // Now that all processes have all the required/computed fields/groups, they
   // have also created any possible internal field (if needed). Notice that some
   // atm proc might have created internal fields already during the set_grids
@@ -606,8 +605,8 @@ void AtmosphereDriver::create_fields()
     m_field_mgr->add_field(f);
   }
 
-  // Now go through the input fields/groups to the atm proc group, as well as
-  // the internal fields/groups, and mark them as part of the RESTART group.
+  // Now go through the input fields/groups to the atm proc group,
+  // and mark them as part of the RESTART group.
   for (const auto& f : m_atm_process_group->get_fields_in()) {
     const auto& fid = f.get_header().get_identifier();
     m_field_mgr->add_to_group(fid, "RESTART");
@@ -621,9 +620,14 @@ void AtmosphereDriver::create_fields()
       }
     }
   }
+  // Internal fields have their group names set by the processes that create them.
+  // Hence, simply add them to all the groups they are marked as part of
   for (const auto& f : m_atm_process_group->get_internal_fields()) {
-    const auto& fid = f.get_header().get_identifier();
-    m_field_mgr->add_to_group(fid, "RESTART");
+    const auto& ftrack = f.get_header().get_tracking();
+    const auto& fid    = f.get_header().get_identifier();
+    for (const auto& gn : ftrack.get_groups_names()) {
+      m_field_mgr->add_to_group(fid, gn);
+    }
   }
 
   auto& driver_options_pl = m_atm_params.sublist("driver_options");
@@ -944,9 +948,16 @@ void AtmosphereDriver::restart_model ()
       // No field needs to be restarted on this grid.
       continue;
     }
-    const auto& restart_group = m_field_mgr->get_group_info("RESTART", gn);
+    const auto& restart_fnames = m_field_mgr->get_group_info("RESTART", gn).m_fields_names;
     std::vector<Field> fields;
-    for (const auto& fn : restart_group.m_fields_names) {
+    for (const auto& fn : restart_fnames) {
+      // If the field has a parent, and the parent is also in the RESTART group,
+      // then skip it, since restarting the parent will restart the child too
+      auto f = m_field_mgr->get_field(fn,gn);
+      auto p = f.get_header().get_parent();
+      if (p and ekat::contains(p->get_tracking().get_groups_names(),"RESTART")) {
+        continue;
+      }
       fields.push_back(m_field_mgr->get_field(fn,gn));
     }
     read_fields_from_file (fields,m_grids_manager->get_grid(gn),filename);

components/eamxx/src/control/tests/ad_tests.yaml

@@ -31,6 +31,7 @@ grids_manager:
   grids_names: ["point_grid"]
   point_grid:
     type: point_grid
+    aliases: ["physics"]
     number_of_global_columns: 24
     number_of_vertical_levels: 3
 ...

components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.cpp

@@ -225,20 +225,20 @@ void HommeDynamics::set_grids (const std::shared_ptr<const GridsManager> grids_m
   // NOTE: 'true' is for 'dynamic' subfield; the idx of the subfield slice will move
   //       during execution (this class will take care of moving it, by calling
   //       reset_subview_idx on each field).
-  add_internal_field (m_helper_fields.at("v_dyn").subfield(1,tl.n0,true));
-  add_internal_field (m_helper_fields.at("vtheta_dp_dyn").subfield(1,tl.n0,true));
-  add_internal_field (m_helper_fields.at("dp3d_dyn").subfield(1,tl.n0,true));
-  add_internal_field (m_helper_fields.at("phi_int_dyn").subfield(1,tl.n0,true));
-  add_internal_field (m_helper_fields.at("ps_dyn").subfield(1,tl.n0,true));
-  add_internal_field (m_helper_fields.at("Qdp_dyn").subfield(1,tl.n0_qdp,true));
+  add_internal_field (m_helper_fields.at("v_dyn").subfield(1,tl.n0,true),{"RESTART"});
+  add_internal_field (m_helper_fields.at("vtheta_dp_dyn").subfield(1,tl.n0,true),{"RESTART"});
+  add_internal_field (m_helper_fields.at("dp3d_dyn").subfield(1,tl.n0,true),{"RESTART"});
+  add_internal_field (m_helper_fields.at("phi_int_dyn").subfield(1,tl.n0,true),{"RESTART"});
+  add_internal_field (m_helper_fields.at("ps_dyn").subfield(1,tl.n0,true),{"RESTART"});
+  add_internal_field (m_helper_fields.at("Qdp_dyn").subfield(1,tl.n0_qdp,true),{"RESTART"});
   if (not params.theta_hydrostatic_mode) {
-    add_internal_field (m_helper_fields.at("w_int_dyn").subfield(1,tl.n0,true));
+    add_internal_field (m_helper_fields.at("w_int_dyn").subfield(1,tl.n0,true),{"RESTART"});
   }
 
   // The output manager pulls from the atm process fields. Add
   // helper fields for the case that a user request output.
   add_internal_field (m_helper_fields.at("omega_dyn"));
-  add_internal_field (m_helper_fields.at("phis_dyn"));
+  add_internal_field (m_helper_fields.at("phis_dyn"),{"RESTART"});
 
   if (not fv_phys_active()) {
     // Dynamics backs out tendencies from the states, and passes those to Homme.
@@ -937,12 +937,15 @@ void HommeDynamics::init_homme_views () {
 }
 
 void HommeDynamics::restart_homme_state () {
-  // Safety checks: internal fields *should* have been restarted (and therefore have a valid timestamp)
+  // Safety checks: RESTART fields *should* have been restarted (and therefore have a valid timestamp)
   for (auto& f : get_internal_fields()) {
-    auto ts = f.get_header().get_tracking().get_time_stamp();
-    EKAT_REQUIRE_MSG(ts.is_valid(),
-        "Error! Found HommeDynamics internal field not restarted.\n"
-        "  - field name: " + f.get_header().get_identifier().name() + "\n");
+    const auto& track = f.get_header().get_tracking();
+    if (ekat::contains(track.get_groups_names(),"RESTART")) {
+      auto ts = track.get_time_stamp();
+      EKAT_REQUIRE_MSG(track.get_time_stamp().is_valid(),
+          "Error! Found HommeDynamics internal field not restarted.\n"
+          "  - field name: " + f.get_header().get_identifier().name() + "\n");
+    }
   }
 
   using TPF = ekat::TeamPolicyFactory<KT::ExeSpace>;
@@ -1320,7 +1323,7 @@ void HommeDynamics::update_pressure(const std::shared_ptr<const AbstractGrid>& g
   const auto p_int_view = get_field_out("p_int",gn).get_view<Pack**>();
   const auto p_mid_view = get_field_out("p_mid",gn).get_view<Pack**>();
 
-  const auto qv_view        = get_field_in("qv").get_view<const Pack**>();
+  const auto qv_view        = get_field_in("qv",gn).get_view<const Pack**>();
   const auto dp_dry_view    = get_field_out("pseudo_density_dry").get_view<Pack**>();
   const auto p_dry_int_view = get_field_out("p_dry_int").get_view<Pack**>();
   const auto p_dry_mid_view = get_field_out("p_dry_mid").get_view<Pack**>();

components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp

@@ -1219,12 +1219,17 @@ void RRTMGPRadiation::run_impl (const double dt) {
 // =========================================================================================
 
 void RRTMGPRadiation::finalize_impl  () {
-  m_gas_concs_k.reset();
-  // Finalize the interface, passing a bool for rank 0
-  // to print info about memory stats on that rank
-  interface_t::rrtmgp_finalize(m_comm.am_i_root());
-
-  finalize_kls();
+  // Guard the finalization, since it would throw if initialize was not called.
+  // This can happen if an atm proc that is inited before RRTMGP throws during init,
+  // and the stack gets destroyed. The driver calls the 'finalize' method on all atm procs
+  if (is_initialized()) {
+    m_gas_concs_k.reset();
+    // Finalize the interface, passing a bool for rank 0
+    // to print info about memory stats on that rank
+    interface_t::rrtmgp_finalize(m_comm.am_i_root());
+
+    finalize_kls();
+  }
 }
 // =========================================================================================