SpectralFieldData: remove timer-based costs tracker from constructor (#6225)

To my understanding, the `SpectralFieldData` constructor is called only
at the beginning of the simulation (and most likely after regrid
operations) to prepare the FFT plans. Therefore, if I am not mistaken,
we should not track the time spent in the constructor for load
balancing, since this routine won't be called again before
redistributing the boxes.

Note that after removing cost tracking from the `SpectralFieldData`
constructor, `lev` becomes an unused parameter.

Author: lucafedeli88

Source/BoundaryConditions/PML.cpp

@@ -792,7 +792,7 @@ PML::PML (const int lev, const BoxArray& grid_ba,
         amrex::Vector<amrex::Real> const v_galilean = WarpX::GetInstance().m_v_galilean;
         amrex::Vector<amrex::Real> const v_comoving_zero = {0., 0., 0.};
         realspace_ba.enclosedCells().grow(nge); // cell-centered + guard cells
-        spectral_solver_fp = std::make_unique<SpectralSolver>(lev, realspace_ba, dm,
+        spectral_solver_fp = std::make_unique<SpectralSolver>(realspace_ba, dm,
             nox_fft, noy_fft, noz_fft, grid_type, v_galilean,
             v_comoving_zero, dx, dt, in_pml, periodic_single_box, update_with_rho,
             fft_do_time_averaging, psatd_solution_type, time_dependency_J, time_dependency_rho, m_dive_cleaning, m_divb_cleaning);
@@ -904,7 +904,7 @@ PML::PML (const int lev, const BoxArray& grid_ba,
             amrex::Vector<amrex::Real> const v_galilean = WarpX::GetInstance().m_v_galilean;
             amrex::Vector<amrex::Real> const v_comoving_zero = {0., 0., 0.};
             realspace_cba.enclosedCells().grow(nge); // cell-centered + guard cells
-            spectral_solver_cp = std::make_unique<SpectralSolver>(lev, realspace_cba, cdm,
+            spectral_solver_cp = std::make_unique<SpectralSolver>(realspace_cba, cdm,
                 nox_fft, noy_fft, noz_fft, grid_type, v_galilean,
                 v_comoving_zero, cdx, dt, in_pml, periodic_single_box, update_with_rho,
                 fft_do_time_averaging, psatd_solution_type, time_dependency_J, time_dependency_rho, m_dive_cleaning, m_divb_cleaning);

Source/FieldSolver/SpectralSolver/SpectralFieldData.H

@@ -144,8 +144,7 @@ class SpectralFieldData
 {
 
     public:
-        SpectralFieldData( int lev,
-                           const amrex::BoxArray& realspace_ba,
+        SpectralFieldData( const amrex::BoxArray& realspace_ba,
                            const SpectralKSpace& k_space,
                            const amrex::DistributionMapping& dm,
                            int n_field_required,

Source/FieldSolver/SpectralSolver/SpectralFieldData.cpp

@@ -129,17 +129,13 @@ SpectralFieldIndex::SpectralFieldIndex (const bool update_with_rho,
 }
 
 /* \brief Initialize fields in spectral space, and FFT plans */
-SpectralFieldData::SpectralFieldData( const int lev,
-                                      const amrex::BoxArray& realspace_ba,
+SpectralFieldData::SpectralFieldData( const amrex::BoxArray& realspace_ba,
                                       const SpectralKSpace& k_space,
                                       const amrex::DistributionMapping& dm,
                                       const int n_field_required,
                                       const bool periodic_single_box):
     m_periodic_single_box{periodic_single_box}
 {
-    amrex::LayoutData<amrex::Real>* cost = WarpX::getCosts(lev);
-    const bool do_costs = WarpXUtilLoadBalance::doCosts(cost, realspace_ba, dm);
-
     const BoxArray& spectralspace_ba = k_space.spectralspace_ba;
 
     // Allocate the arrays that contain the fields in spectral space
@@ -177,12 +173,6 @@ SpectralFieldData::SpectralFieldData( const int lev,
     // Loop over boxes and allocate the corresponding plan
     // for each box owned by the local MPI proc
     for ( MFIter mfi(spectralspace_ba, dm); mfi.isValid(); ++mfi ){
-        if (do_costs)
-        {
-            amrex::Gpu::synchronize();
-        }
-        auto wt = static_cast<amrex::Real>(amrex::second());
-
         // Note: the size of the real-space box and spectral-space box
         // differ when using real-to-complex FFT. When initializing
         // the FFT plan, the valid dimensions are those of the real-space box.
@@ -197,13 +187,6 @@ SpectralFieldData::SpectralFieldData( const int lev,
             fft_size, tmpRealField[mfi].dataPtr(),
             reinterpret_cast<ablastr::math::anyfft::Complex*>( tmpSpectralField[mfi].dataPtr()),
             ablastr::math::anyfft::direction::C2R, AMREX_SPACEDIM);
-
-        if (do_costs)
-        {
-            amrex::Gpu::synchronize();
-            wt = static_cast<amrex::Real>(amrex::second()) - wt;
-            amrex::HostDevice::Atomic::Add( &(*cost)[mfi.index()], wt);
-        }
     }
 }
 

Source/FieldSolver/SpectralSolver/SpectralSolver.H

@@ -44,7 +44,6 @@ class SpectralSolver
          * for the discrete field update equations, and prepare the structures that store
          * the fields in spectral space.
          *
-         * \param[in] lev mesh refinement level
          * \param[in] realspace_ba BoxArray in real space
          * \param[in] dm DistributionMapping for the given BoxArray
          * \param[in] norder_x spectral order along x
@@ -73,8 +72,7 @@ class SpectralSolver
          * \param[in] divb_cleaning whether to use div(B) cleaning to account for errors in
          *                          div(B) = 0 law (new field G in the update equations)
          */
-        SpectralSolver (int lev,
-                        const amrex::BoxArray& realspace_ba,
+        SpectralSolver (const amrex::BoxArray& realspace_ba,
                         const amrex::DistributionMapping& dm,
                         int norder_x,
                         int norder_y,

Source/FieldSolver/SpectralSolver/SpectralSolver.cpp

@@ -24,7 +24,6 @@
 #if WARPX_USE_FFT
 
 SpectralSolver::SpectralSolver (
-                const int lev,
                 const amrex::BoxArray& realspace_ba,
                 const amrex::DistributionMapping& dm,
                 const int norder_x,
@@ -110,7 +109,7 @@ SpectralSolver::SpectralSolver (
     }
 
     // - Initialize arrays for fields in spectral space + FFT plans
-    field_data = SpectralFieldData(lev, realspace_ba, k_space, dm,
+    field_data = SpectralFieldData(realspace_ba, k_space, dm,
                                    m_spectral_index.n_fields, periodic_single_box);
 }
 

Source/WarpX.cpp

@@ -3061,8 +3061,7 @@ void WarpX::AllocLevelSpectralSolver (amrex::Vector<std::unique_ptr<SpectralSolv
         solver_dt /= 2.;
     }
 
-    auto pss = std::make_unique<SpectralSolver>(lev,
-                                                realspace_ba,
+    auto pss = std::make_unique<SpectralSolver>(realspace_ba,
                                                 dm,
                                                 nox_fft,
                                                 noy_fft,