Fix RAW (read after write) race hazard in theta-l_kokkos limiter

Also remove (slow) profiling calls

Author: amametjanov

components/homme/src/theta-l_kokkos/cxx/CaarFunctorImpl.hpp

@@ -343,8 +343,6 @@ struct CaarFunctorImpl {
 
     set_rk_stage_data(data);
 
-    profiling_resume();
-
     GPTLstart("caar compute");
     int nerr;
     Kokkos::parallel_reduce("caar loop pre-boundary exchange", m_policy_pre, *this, nerr);
@@ -367,7 +365,6 @@ struct CaarFunctorImpl {
 
     limiter.run(data.np1);
 
-    profiling_pause();
   }
 
   KOKKOS_INLINE_FUNCTION

components/homme/src/theta-l_kokkos/cxx/LimiterFunctor.hpp

@@ -99,15 +99,11 @@ struct LimiterFunctor {
 
   void run (const int& tl)
   {
-    profiling_resume();
-
     GPTLstart("caar limiter");
     m_np1 = tl;
     Kokkos::parallel_for("caar loop dp3d limiter", m_policy_dp3d_lim, *this);
     Kokkos::fence();
     GPTLstop("caar limiter");
-
-    profiling_pause();
   }
 
   KOKKOS_INLINE_FUNCTION
@@ -147,6 +143,7 @@ struct LimiterFunctor {
 #endif
         result = result<=diff_as_real(k) ? result : diff_as_real(k);
       }, reducer);
+      kv.team_barrier();
 
       auto vtheta_dp = Homme::subview(m_state.m_vtheta_dp,kv.ie,m_np1,igp,jgp);
 
@@ -168,8 +165,6 @@ struct LimiterFunctor {
           });
         }
 
-        kv.team_barrier();
-
         // This loop must be done over physical levels, unless we implement
         // masks, like it has been done in the E3SM/scream project
         Real mass_new = 0.0;
@@ -194,6 +189,7 @@ struct LimiterFunctor {
           vtheta_dp(ilev) *= dp(ilev);
         });
       } //end of min_diff < 0
+      kv.team_barrier();
 
       Kokkos::parallel_for(Kokkos::ThreadVectorRange(kv.team,NUM_LEV),
                            [&](const int ilev) {