Heuristic load balance based on number of particles and number of cells (#737)

mrowan137 · MaxThevenet · web-flow · commit c8b1d2175fce · 2020-03-02T09:26:45.000-08:00
* Issue #713: Load Balance w.r.t. number of cells + particles README: Update Badged to `master` (#725) * README: Update Badged to `master` Update the badges to check the `master` branch as development branch. * Docs: More dev->master updates - release workflow - contribution guide link Galilean PSATD with shift (#704) * Read Galilean velocity * Prepare structures for Galilean solver * Started implementing Galilean equations * Analytical limits for X1, X2, X3, X4 coefficients added * Slight changes added * Added Galilean position pusher * Scale galilean velocity * Remove unneeded Abort * Fix Galilean pusher * Allocate Theta2 array * Fix definition of coefficients * Increase guard cells for Galilean * Add guard cell in particle exchange * Type corrected * v_gal added to warpx_current_deposition * v_gal added to WarpXParticleContainer.H * Bug fixed - update particle x-position over one time step * Fix issues with merge from dev * Preparation for merging dev into galilean. * Adding galilean shift * Implemented galilean shift * Changed method's name from GalileanShift to ShiftGalileanBoundary * Added doxygen string for ShiftGalileanBoundary * Removed never used method LowerCornerWithCentering * Removed temporary comments * Removed dt as a variable from DepositCharge method and its dependencies * Converted tab to spaces * Removed EOL white space * Add documentation and automated tests * Fix compilation error * Add automated test * Update automated test * Removed temporary used galilean shift * Removed temporary used particle's push for Galilean PSATD * Removed unused statement * Remove EOL white space. * Added zero shift for LowerCorner in RZ geometry * Minor changes to Galilean implementation * Modifications for GPU * Fix typo Co-authored-by: Remi Lehe <remi.lehe@normalesup.org> [mini-PR] when a cufft error occurs, print a meaningful error message (#728) * added method to translate cufft errors * fixed style * bug fixing avoid duplicate tests and plot less often (#726) * avoid duplicate tests and plot less often * fix tests I broke when trying to save plotfiles doc install yt on Summit (#729) * doc install yt on Summit * eol Do not use local Redistribute for electrostatic solver (#731) Add Reset Random Seed Feature (#717) * Add ResetRandomSeed * Add doc * Modify and change location of the code. * Small fix * Try to fix an alert * Try to fix an alert * Modify based on suggestions * Use INT_MAX * Modify based on suggestions. * Modify based on suggestions. openPMD: warn if step is already written (#718) * making sure iterations are written at most once. * prints a warning when iteration is written more than once writting is not stopped * Fixed tabs * included <iostream> as requested by Axel Minor refactoring of space-charge calculation (#732) Improve clarity and documentation Minor Update GNUmakefile Update based on comments Update GNU Makefile Formatting Formatting Formatting Formatting Remove unneeded function Removed unneeded function Formatting Formatting Formatting Whitespace Minor Formatting Formatting Formatting Formatting Formatting Formatting Formatting whitespace Formatting Minor Formatting Remove unneeded template function Change import Minor Formatting Remove unused variable Formatting Update Source/WarpX.H Co-Authored-By: MaxThevenet <mthevenet@lbl.gov> Update Source/Parallelization/WarpXRegrid.cpp Co-Authored-By: MaxThevenet <mthevenet@lbl.gov> Remove `n_particles` and `n_cells` Update Source/WarpX.H Co-Authored-By: MaxThevenet <mthevenet@lbl.gov> Revert clear costs in case of edge case Update to use new load_balance_api in AMReX Tabs Minor * minor: indentation in Source/WarpX.H Co-authored-by: MaxThevenet <mthevenet@lbl.gov>
diff --git a/Source/Diagnostics/FieldIO.cpp b/Source/Diagnostics/FieldIO.cpp
@@ -506,10 +506,11 @@ WarpX::AverageAndPackFields ( Vector<std::string>& varnames,
                               amrex::Vector<MultiFab>& mf_avg, const int ngrow) const
 {
     // Count how many different fields should be written (ncomp)
+    MultiFab* cost = WarpX::getCosts(0);
     int ncomp = fields_to_plot.size()
         + static_cast<int>(plot_finepatch)*6
         + static_cast<int>(plot_crsepatch)*6
-        + static_cast<int>(costs[0] != nullptr and plot_costs);
+        + static_cast<int>(cost != nullptr and plot_costs);
 
     // Add in the RZ modes
     if (n_rz_azimuthal_modes > 1) {
@@ -723,13 +724,15 @@ WarpX::AverageAndPackFields ( Vector<std::string>& varnames,
             dcomp += 3;
         }
 
-        if (costs[0] != nullptr and plot_costs)
+        if (WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers)
         {
-            AverageAndPackScalarField( mf_avg[lev], *costs[lev], dcomp, ngrow );
-            if(lev==0) varnames.push_back("costs");
-            dcomp += 1;
+            if (costs[0] != nullptr and plot_costs)
+            {
+                AverageAndPackScalarField( mf_avg[lev], *costs[lev], dcomp, ngrow );
+                if(lev==0) varnames.push_back("costs");
+                dcomp += 1;
+            }
         }
-
         BL_ASSERT(dcomp == ncomp);
     } // end loop over levels of refinement
 
diff --git a/Source/Diagnostics/WarpXIO.cpp b/Source/Diagnostics/WarpXIO.cpp
@@ -187,9 +187,11 @@ WarpX::WriteCheckPointFile() const
             pml[lev]->CheckPoint(amrex::MultiFabFileFullPrefix(lev, checkpointname, level_prefix, "pml"));
         }
 
-        if (costs[lev]) {
-            VisMF::Write(*costs[lev],
-                         amrex::MultiFabFileFullPrefix(lev, checkpointname, level_prefix, "costs"));
+        if (WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) {
+            if (costs[lev]) {
+                VisMF::Write(*costs[lev],
+                             amrex::MultiFabFileFullPrefix(lev, checkpointname, level_prefix, "costs"));
+            }
         }
     }
 
@@ -382,13 +384,15 @@ WarpX::InitFromCheckpoint ()
             }
         }
 
-        if (costs[lev]) {
-            const auto& cost_mf_name =
+        if (WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) {
+            if (costs[lev]) {
+                const auto& cost_mf_name =
                 amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "costs");
-            if (VisMF::Exist(cost_mf_name)) {
-                VisMF::Read(*costs[lev], cost_mf_name);
-            } else {
-                costs[lev]->setVal(0.0);
+                if (VisMF::Exist(cost_mf_name)) {
+                    VisMF::Read(*costs[lev], cost_mf_name);
+                } else {
+                    costs[lev]->setVal(0.0);
+                }
             }
         }
     }
diff --git a/Source/Evolve/WarpXEvolveEM.cpp b/Source/Evolve/WarpXEvolveEM.cpp
@@ -38,7 +38,7 @@ WarpX::EvolveEM (int numsteps)
     static int last_check_file_step = 0;
     static int last_insitu_step = 0;
 
-    if (do_compute_max_step_from_zmax){
+    if (do_compute_max_step_from_zmax) {
         computeMaxStepBoostAccelerator(geom[0]);
     }
 
@@ -61,25 +61,36 @@ WarpX::EvolveEM (int numsteps)
         if (warpx_py_beforestep) warpx_py_beforestep();
 #endif
 
-        if (costs[0] != nullptr)
-        {
+        MultiFab* cost = WarpX::getCosts(0);
+        amrex::Vector<amrex::Real>* cost_heuristic = WarpX::getCostsHeuristic(0);
+        if (cost != nullptr || cost_heuristic != nullptr) {
 #ifdef WARPX_USE_PSATD
             amrex::Abort("LoadBalance for PSATD: TODO");
 #endif
-
             if (step > 0 && (step+1) % load_balance_int == 0)
             {
                 LoadBalance();
                 // Reset the costs to 0
-                for (int lev = 0; lev <= finest_level; ++lev) {
-                    costs[lev]->setVal(0.0);
+                for (int lev = 0; lev <= finest_level; ++lev)
+                {
+                    if (WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers)
+                    {
+                        costs[lev]->setVal(0.0);
+                    } else if (WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Heuristic)
+                    {
+                        costs_heuristic[lev]->assign((*costs_heuristic[lev]).size(), 0.0);
+                    }
                 }
             }
-
-            for (int lev = 0; lev <= finest_level; ++lev) {
-                // Perform running average of the costs
-                // (Giving more importance to most recent costs)
-                (*costs[lev].get()).mult( (1. - 2./load_balance_int) );
+            for (int lev = 0; lev <= finest_level; ++lev)
+            {
+                MultiFab* cost = WarpX::getCosts(lev);
+                if (cost)
+                {
+                    // Perform running average of the costs
+                    // (Giving more importance to most recent costs)
+                    cost->mult( (1. - 2./load_balance_int) );
+                }
             }
         }
 
@@ -92,13 +103,13 @@ WarpX::EvolveEM (int numsteps)
             FillBoundaryB(guard_cells.ng_alloc_EB, guard_cells.ng_Extra);
             UpdateAuxilaryData();
             // on first step, push p by -0.5*dt
-            for (int lev = 0; lev <= finest_level; ++lev) {
+            for (int lev = 0; lev <= finest_level; ++lev)
+            {
                 mypc->PushP(lev, -0.5*dt[lev],
                             *Efield_aux[lev][0],*Efield_aux[lev][1],*Efield_aux[lev][2],
                             *Bfield_aux[lev][0],*Bfield_aux[lev][1],*Bfield_aux[lev][2]);
             }
             is_synchronized = false;
-
         } else {
             // Beyond one step, we have E^{n} and B^{n}.
             // Particles have p^{n-1/2} and x^{n}.
diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp
@@ -262,7 +262,7 @@ WarpX::EvolveE (int lev, PatchType patch_type, amrex::Real a_dt)
         F  = F_cp[lev].get();
     }
 
-    MultiFab* cost = costs[lev].get();
+    MultiFab* cost = WarpX::getCosts(lev);
     const IntVect& rr = (lev > 0) ? refRatio(lev-1) : IntVect::TheUnitVector();
 
     // xmin is only used by the kernel for cylindrical geometry,
diff --git a/Source/FieldSolver/WarpX_QED_Field_Pushers.cpp b/Source/FieldSolver/WarpX_QED_Field_Pushers.cpp
@@ -83,7 +83,7 @@ WarpX::Hybrid_QED_Push (int lev, PatchType patch_type, Real a_dt)
         Bz = Bfield_cp[lev][2].get();
     }
 
-    MultiFab* cost = costs[lev].get();
+    MultiFab* cost = WarpX::getCosts(lev);
     const IntVect& rr = (lev > 0) ? refRatio(lev-1) : IntVect::TheUnitVector();
 
     // xmin is only used by the kernel for cylindrical geometry,
diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp
@@ -12,6 +12,7 @@
 #include "Filter/NCIGodfreyFilter.H"
 #include "Parser/GpuParser.H"
 #include "Utils/WarpXUtil.H"
+#include "Utils/WarpXAlgorithmSelection.H"
 
 #include <AMReX_ParallelDescriptor.H>
 #include <AMReX_ParmParse.H>
@@ -420,8 +421,16 @@ WarpX::InitLevelData (int lev, Real /*time*/)
         rho_cp[lev]->setVal(0.0);
     }
 
-    if (costs[lev]) {
-        costs[lev]->setVal(0.0);
+    if (WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) {
+        if (costs[lev]) {
+            costs[lev]->setVal(0.0);
+        }
+    } else if (WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Heuristic) {
+        if (costs_heuristic[lev]) {
+            std::fill((*costs_heuristic[lev]).begin(),
+                      (*costs_heuristic[lev]).end(),
+                      0.0);
+        }
     }
 }
 
diff --git a/Source/Parallelization/WarpXRegrid.cpp b/Source/Parallelization/WarpXRegrid.cpp
@@ -7,6 +7,7 @@
  * License: BSD-3-Clause-LBNL
  */
 #include <WarpX.H>
+#include <WarpXAlgorithmSelection.H>
 #include <AMReX_BLProfiler.H>
 
 using namespace amrex;
@@ -17,6 +18,18 @@ WarpX::LoadBalance ()
     WARPX_PROFILE_REGION("LoadBalance");
     WARPX_PROFILE("WarpX::LoadBalance()");
 
+    if (WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers)
+    {
+        LoadBalanceTimers();
+    } else if (WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Heuristic)
+    {
+        LoadBalanceHeuristic();
+    }
+}
+
+void
+WarpX::LoadBalanceTimers ()
+{
     AMREX_ALWAYS_ASSERT(costs[0] != nullptr);
 
     const int nLevels = finestLevel();
@@ -30,7 +43,36 @@ WarpX::LoadBalance ()
             : DistributionMapping::makeKnapSack(*costs[lev], nmax);
         RemakeLevel(lev, t_new[lev], boxArray(lev), newdm);
     }
+    mypc->Redistribute();
+}
+
+void
+WarpX::LoadBalanceHeuristic ()
+{
+    AMREX_ALWAYS_ASSERT(costs_heuristic[0] != nullptr);
+    WarpX::ComputeCostsHeuristic();
+
+    const int nLevels = finestLevel();
+    for (int lev = 0; lev <= nLevels; ++lev)
+    {
+#ifdef AMREX_USE_MPI
+        // Parallel reduce the costs_heurisitc
+        amrex::Vector<Real>::iterator it = (*costs_heuristic[lev]).begin();
+        amrex::Real* itAddr = &(*it);
+        ParallelAllReduce::Sum(itAddr,
+                               costs_heuristic[lev]->size(),
+                               ParallelContext::CommunicatorSub());
+#endif
+        const amrex::Real nboxes = costs_heuristic[lev]->size();
+        const amrex::Real nprocs = ParallelContext::NProcsSub();
+        const int nmax = static_cast<int>(std::ceil(nboxes/nprocs*load_balance_knapsack_factor));
 
+        const DistributionMapping newdm = (load_balance_with_sfc)
+            ? DistributionMapping::makeSFC(*costs_heuristic[lev], boxArray(lev), false)
+            : DistributionMapping::makeKnapSack(*costs_heuristic[lev], nmax);
+
+        RemakeLevel(lev, t_new[lev], boxArray(lev), newdm);
+    }
     mypc->Redistribute();
 }
 
@@ -47,7 +89,6 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi
 #endif // WARPX_DO_ELECTROSTATIC
 
         // Fine patch
-
         const auto& period = Geom(lev).periodicity();
         for (int idim=0; idim < 3; ++idim)
         {
@@ -98,7 +139,6 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi
         }
 
         // Aux patch
-
         if (lev == 0 && Bfield_aux[0][0]->ixType() == Bfield_fp[0][0]->ixType())
         {
             for (int idim = 0; idim < 3; ++idim) {
@@ -223,15 +263,57 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi
             }
         }
 
-        if (costs[lev] != nullptr) {
-            costs[lev].reset(new MultiFab(costs[lev]->boxArray(), dm, 1, 0));
-            costs[lev]->setVal(0.0);
+        if (WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers)
+        {
+            if (costs[lev] != nullptr)
+            {
+                costs[lev].reset(new MultiFab(costs[lev]->boxArray(), dm, 1, 0));
+                costs[lev]->setVal(0.0);
+            }
+        } else if (WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Heuristic)
+        {
+            if (costs_heuristic[lev] != nullptr)
+            {
+                costs_heuristic[lev].reset(new amrex::Vector<Real>);
+                const int nboxes = Efield_fp[lev][0].get()->size();
+                costs_heuristic[lev]->resize(nboxes, 0.0); // Initializes to 0.0?
+            }
         }
 
         SetDistributionMap(lev, dm);
-    }
-    else
+
+    } else
     {
         amrex::Abort("RemakeLevel: to be implemented");
     }
 }
+
+void
+WarpX::ComputeCostsHeuristic ()
+{
+    for (int lev = 0; lev <= finest_level; ++lev)
+    {
+        auto & mypc = WarpX::GetInstance().GetPartContainer();
+        auto nSpecies = mypc.nSpecies();
+
+        // Species loop
+        for (int i_s = 0; i_s < nSpecies; ++i_s)
+        {
+            auto & myspc = mypc.GetParticleContainer(i_s);
+
+            // Particle loop
+            for (WarpXParIter pti(myspc, lev); pti.isValid(); ++pti)
+            {
+                (*costs_heuristic[lev])[pti.index()] += costs_heuristic_particles_wt*pti.numParticles();
+            }
+        }
+
+        //Cell loop
+        MultiFab* Ex = Efield_fp[lev][0].get();
+        for (MFIter mfi(*Ex, false); mfi.isValid(); ++mfi)
+        {
+            const Box& gbx = mfi.growntilebox();
+            (*costs_heuristic[lev])[mfi.index()] += costs_heuristic_cells_wt*gbx.numPts();
+        }
+    } // for (int lev ...)
+} // WarpX::ComputeCostsHeuristic
diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp
@@ -156,9 +156,7 @@ void PhysicalParticleContainer::InitData()
     // Init ionization module here instead of in the PhysicalParticleContainer
     // constructor because dt is required
     if (do_field_ionization) {InitIonizationModule();}
-
     AddParticles(0); // Note - add on level 0
-
     Redistribute();  // We then redistribute
 }
 
@@ -1135,6 +1133,7 @@ PhysicalParticleContainer::Evolve (int lev,
     BL_ASSERT(OnSameGrids(lev,jx));
 
     MultiFab* cost = WarpX::getCosts(lev);
+
     const iMultiFab* current_masks = WarpX::CurrentBufferMasks(lev);
     const iMultiFab* gather_masks = WarpX::GatherBufferMasks(lev);
 
diff --git a/Source/Utils/WarpXAlgorithmSelection.H b/Source/Utils/WarpXAlgorithmSelection.H
@@ -47,6 +47,16 @@ struct GatheringAlgo {
     };
 };
 
+/** Strategy to compute weights for use in load balance.
+ */
+struct LoadBalanceCostsUpdateAlgo {
+    enum {
+        Timers = 0,   //!< load balance according to in-code timer-based weights (i.e., with  `costs`)
+        Heuristic = 1 /**< load balance according to weights computed from number of cells
+                             and number of particles per box (i.e., with `costs_heuristic`)*/
+    };
+};
+
 int
 GetAlgorithmInteger( amrex::ParmParse& pp, const char* pp_search_key );
 
diff --git a/Source/Utils/WarpXAlgorithmSelection.cpp b/Source/Utils/WarpXAlgorithmSelection.cpp
@@ -52,6 +52,12 @@ const std::map<std::string, int> gathering_algo_to_int = {
     {"default",             GatheringAlgo::EnergyConserving }
 };
 
+const std::map<std::string, int> load_balance_costs_update_algo_to_int = {
+    {"timers",    LoadBalanceCostsUpdateAlgo::Timers },
+    {"heuristic", LoadBalanceCostsUpdateAlgo::Heuristic },
+    {"default",   LoadBalanceCostsUpdateAlgo::Timers }
+};
+
 
 int
 GetAlgorithmInteger( amrex::ParmParse& pp, const char* pp_search_key ){
@@ -74,13 +80,15 @@ GetAlgorithmInteger( amrex::ParmParse& pp, const char* pp_search_key ){
         algo_to_int = charge_deposition_algo_to_int;
     } else if (0 == std::strcmp(pp_search_key, "field_gathering")) {
         algo_to_int = gathering_algo_to_int;
+    } else if (0 == std::strcmp(pp_search_key, "load_balance_costs_update")) {
+        algo_to_int = load_balance_costs_update_algo_to_int;
     } else {
         std::string pp_search_string = pp_search_key;
         amrex::Abort("Unknown algorithm type: " + pp_search_string);
     }
 
     // Check if the user-input is a valid key for the dictionary
-    if (algo_to_int.count(algo) == 0){
+    if (algo_to_int.count(algo) == 0) {
         // Not a valid key ; print error message
         std::string pp_search_string = pp_search_key;
         std::string error_message = "Invalid string for algo." + pp_search_string
diff --git a/Source/WarpX.H b/Source/WarpX.H
diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp

Original file line number	Diff line number	Diff line change
`@@ -187,9 +187,11 @@ WarpX::WriteCheckPointFile() const`
`187`	`187`	`pml[lev]->CheckPoint(amrex::MultiFabFileFullPrefix(lev, checkpointname, level_prefix, "pml"));`
`188`	`188`	`}`
`189`	`189`
`190`		`- if (costs[lev]) {`
`191`		`- VisMF::Write(*costs[lev],`
`192`		`- amrex::MultiFabFileFullPrefix(lev, checkpointname, level_prefix, "costs"));`
	`190`	`+ if (WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) {`
	`191`	`+ if (costs[lev]) {`
	`192`	`+ VisMF::Write(*costs[lev],`
	`193`	`+ amrex::MultiFabFileFullPrefix(lev, checkpointname, level_prefix, "costs"));`
	`194`	`+ }`
`193`	`195`	`}`
`194`	`196`	`}`
`195`	`197`
`@@ -382,13 +384,15 @@ WarpX::InitFromCheckpoint ()`
`382`	`384`	`}`
`383`	`385`	`}`
`384`	`386`
`385`		`- if (costs[lev]) {`
`386`		`- const auto& cost_mf_name =`
	`387`	`+ if (WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) {`
	`388`	`+ if (costs[lev]) {`
	`389`	`+ const auto& cost_mf_name =`
`387`	`390`	`amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "costs");`
`388`		`- if (VisMF::Exist(cost_mf_name)) {`
`389`		`- VisMF::Read(*costs[lev], cost_mf_name);`
`390`		`- } else {`
`391`		`- costs[lev]->setVal(0.0);`
	`391`	`+ if (VisMF::Exist(cost_mf_name)) {`
	`392`	`+ VisMF::Read(*costs[lev], cost_mf_name);`
	`393`	`+ } else {`
	`394`	`+ costs[lev]->setVal(0.0);`
	`395`	`+ }`
`392`	`396`	`}`
`393`	`397`	`}`
`394`	`398`	`}`
Original file line number	Diff line number	Diff line change
`@@ -262,7 +262,7 @@ WarpX::EvolveE (int lev, PatchType patch_type, amrex::Real a_dt)`
`262`	`262`	`F = F_cp[lev].get();`
`263`	`263`	`}`
`264`	`264`
`265`		`- MultiFab* cost = costs[lev].get();`
	`265`	`+ MultiFab* cost = WarpX::getCosts(lev);`
`266`	`266`	`const IntVect& rr = (lev > 0) ? refRatio(lev-1) : IntVect::TheUnitVector();`
`267`	`267`
`268`	`268`	`// xmin is only used by the kernel for cylindrical geometry,`
Original file line number	Diff line number	Diff line change
`@@ -83,7 +83,7 @@ WarpX::Hybrid_QED_Push (int lev, PatchType patch_type, Real a_dt)`
`83`	`83`	`Bz = Bfield_cp[lev][2].get();`
`84`	`84`	`}`
`85`	`85`
`86`		`- MultiFab* cost = costs[lev].get();`
	`86`	`+ MultiFab* cost = WarpX::getCosts(lev);`
`87`	`87`	`const IntVect& rr = (lev > 0) ? refRatio(lev-1) : IntVect::TheUnitVector();`
`88`	`88`
`89`	`89`	`// xmin is only used by the kernel for cylindrical geometry,`