created an ImplicitOptions struct to pass into Evolve(). (BLAST-WarpX#6116)

In future PRs, the PhysicalParticleContainer::Evolve() routine will
require more parameters specific to the implicit methods. To avoid
bloating the list of parameter passed into Evolve(), this PR creates an
ImplicitOptions structure to pass into Evolve() that owns all of the
implicit options needed by the particle push and deposit routines.

This allows direct and dynamic control of the what goes on in the
Evolve() routine from the ImplicitSolver class.

Author: JustinRayAngus

Source/Diagnostics/WarpXIO.cpp

@@ -410,8 +410,6 @@ WarpX::InitFromCheckpoint ()
     if (m_implicit_solver) {
 
         m_implicit_solver->Define(this);
-        m_implicit_solver->GetParticleSolverParams( max_particle_its_in_implicit_scheme,
-                                                    particle_tol_in_implicit_scheme );
         m_implicit_solver->CreateParticleAttributes();
     }
 

Source/Evolve/WarpXEvolve.cpp

@@ -1212,8 +1212,7 @@ void
 WarpX::PushParticlesandDeposit (
     amrex::Real cur_time,
     bool skip_current,
-    bool deposit_mass_matrices,
-    PushType push_type
+    ImplicitOptions const * implicit_options
 )
 {
     // Evolve particles to p^{n+1/2} and x^{n+1}
@@ -1224,8 +1223,7 @@ WarpX::PushParticlesandDeposit (
             cur_time,
             DtType::Full,
             skip_current,
-            deposit_mass_matrices,
-            push_type
+            implicit_options
         );
     }
 }
@@ -1236,8 +1234,7 @@ WarpX::PushParticlesandDeposit (
     amrex::Real cur_time,
     DtType a_dt_type,
     bool skip_current,
-    bool deposit_mass_matrices,
-    PushType push_type
+    ImplicitOptions const * implicit_options
 )
 {
     using ablastr::fields::Direction;
@@ -1266,8 +1263,7 @@ WarpX::PushParticlesandDeposit (
         dt[lev],
         a_dt_type,
         skip_current,
-        deposit_mass_matrices,
-        push_type
+        implicit_options
     );
 
     if (! skip_current) {

Source/FieldSolver/ImplicitSolvers/ImplicitOptions.H

@@ -0,0 +1,15 @@
+#ifndef IMPLICIT_OPTIONS_H_
+#define IMPLICIT_OPTIONS_H_
+
+#include <AMReX_Particles.H>
+using namespace amrex::literals;
+
+struct ImplicitOptions {
+    int max_particle_iterations = 21;
+    amrex::ParticleReal particle_tolerance = 1.0e-10_prt;
+    bool deposit_mass_matrices = false;
+    bool linear_stage_of_jfnk = false;
+    int nonlinear_iteration = 0;
+};
+
+#endif

Source/FieldSolver/ImplicitSolvers/ImplicitSolver.H

@@ -7,6 +7,8 @@
 #ifndef Implicit_Solver_H_
 #define Implicit_Solver_H_
 
+#include "FieldSolver/ImplicitSolvers/ImplicitOptions.H"
+
 #include "FieldSolver/ImplicitSolvers/WarpXSolverVec.H"
 #include "NonlinearSolvers/NonlinearSolverLibrary.H"
 
@@ -50,13 +52,6 @@ public:
 
     virtual void PrintParameters () const = 0;
 
-    void GetParticleSolverParams (int&  a_max_particle_iter,
-                                  amrex::ParticleReal&  a_particle_tol ) const
-    {
-        a_max_particle_iter = m_max_particle_iterations;
-        a_particle_tol = m_particle_tolerance;
-    }
-
     void CreateParticleAttributes () const;
 
     /**
@@ -145,7 +140,7 @@ protected:
      * \brief tolerance used by the iterative method used to obtain a self-consistent
      *  update of the particle positions and velocities for given E and B on the grid
      */
-    amrex::ParticleReal m_particle_tolerance = 1.0e-10;
+    amrex::ParticleReal m_particle_tolerance = amrex::ParticleReal(1.0e-10);
 
     /**
      * \brief maximum iterations for the iterative method used to obtain a self-consistent

Source/FieldSolver/ImplicitSolvers/ImplicitSolver.cpp

@@ -1,7 +1,6 @@
 #include "ImplicitSolver.H"
 #include "Fields.H"
 #include "WarpX.H"
-#include "Evolve/WarpXPushType.H"
 #include "Particles/MultiParticleContainer.H"
 #include "Utils/WarpXAlgorithmSelection.H"
 
@@ -687,7 +686,6 @@ void ImplicitSolver::PreRHSOp ( const amrex::Real  a_cur_time,
     BL_PROFILE("ImplicitSolver::PreRHSOp()");
 
     using warpx::fields::FieldType;
-    amrex::ignore_unused( a_nl_iter );
 
     if (m_WarpX->use_filter) {
         int finest_level = 0;
@@ -698,12 +696,18 @@ void ImplicitSolver::PreRHSOp ( const amrex::Real  a_cur_time,
     // particle velocities by dt, then take average of old and new v,
     // deposit currents, giving J at n+1/2
     // This uses Efield_fp and Bfield_fp, the field at n+1/2 from the previous iteration.
-    const PushType push_type = PushType::Implicit;
     const bool skip_current = false;
 
+    // Set the implict solver options for particles and setting the current density
+    ImplicitOptions options;
+    options.nonlinear_iteration = a_nl_iter;
+    options.max_particle_iterations = m_max_particle_iterations;
+    options.particle_tolerance = m_particle_tolerance;
+    options.linear_stage_of_jfnk = a_from_jacobian;
+
     if (m_use_mass_matrices && !a_from_jacobian) { // Called from non-linear stage of JFNK and using mass matrices
-        bool deposit_mass_matrices = true;
-        m_WarpX->PushParticlesandDeposit(a_cur_time, skip_current, deposit_mass_matrices, push_type);
+        options.deposit_mass_matrices = true;
+        m_WarpX->PushParticlesandDeposit(a_cur_time, skip_current, &options);
         if (m_use_mass_matrices_jacobian) { SaveEandJ(); }
         if (m_use_mass_matrices_pc) {
            SyncMassMatricesPCAndApplyBCs();
@@ -715,8 +719,8 @@ void ImplicitSolver::PreRHSOp ( const amrex::Real  a_cur_time,
         ComputeJfromMassMatrices();
     }
     else {  // Conventional particle-suppressed JFNK
-        bool deposit_mass_matrices = false;
-        m_WarpX->PushParticlesandDeposit(a_cur_time, skip_current, deposit_mass_matrices, push_type);
+        options.deposit_mass_matrices = false;
+        m_WarpX->PushParticlesandDeposit(a_cur_time, skip_current, &options);
     }
 
     // Apply BCs to J and communicate

Source/Initialization/WarpXInitData.cpp

@@ -925,8 +925,6 @@ WarpX::InitFromScratch ()
     if (m_implicit_solver) {
 
         m_implicit_solver->Define(this);
-        m_implicit_solver->GetParticleSolverParams( max_particle_its_in_implicit_scheme,
-                                                    particle_tol_in_implicit_scheme );
         m_implicit_solver->CreateParticleAttributes();
     }
 

Source/Particles/LaserParticleContainer.H

@@ -14,6 +14,8 @@
 #include "Laser/LaserProfiles.H"
 #include "WarpXParticleContainer.H"
 
+#include "FieldSolver/ImplicitSolvers/ImplicitOptions.H"
+
 #include <AMReX_Extension.H>
 #include <AMReX_Random.H>
 #include <AMReX_REAL.H>
@@ -68,8 +70,8 @@ public:
                  int lev,
                  const std::string& current_fp_string,
                  amrex::Real t, amrex::Real dt, DtType a_dt_type=DtType::Full,
-                 bool skip_deposition=false, bool deposit_mass_matrices=false,
-                 PushType push_type=PushType::Explicit) final;
+                 bool skip_deposition=false,
+                 ImplicitOptions const * implicit_options = nullptr) final;
 
     void PushP (int lev, amrex::Real dt,
                         const amrex::MultiFab& ,

Source/Particles/LaserParticleContainer.cpp

@@ -563,7 +563,7 @@ LaserParticleContainer::Evolve (ablastr::fields::MultiFabRegister& fields,
                                 int lev,
                                 const std::string& current_fp_string,
                                 Real t, Real dt, DtType /*a_dt_type*/, bool skip_deposition,
-                                bool /*deposit_mass_matrices*/, PushType push_type)
+                                ImplicitOptions const * implicit_options)
 {
     using ablastr::fields::Direction;
     using warpx::fields::FieldType;
@@ -573,6 +573,8 @@ LaserParticleContainer::Evolve (ablastr::fields::MultiFabRegister& fields,
 
     if (!m_enabled) { return; }
 
+    const PushType push_type = (implicit_options == nullptr) ? PushType::Explicit : PushType::Implicit;
+
     Real t_lab = t;
     if (WarpX::gamma_boost > 1) {
         // Convert time from the boosted to the lab-frame

Source/Particles/MultiParticleContainer.H

@@ -12,6 +12,7 @@
 #define WARPX_ParticleContainer_H_
 
 #include "MultiParticleContainer_fwd.H"
+#include "FieldSolver/ImplicitSolvers/ImplicitOptions.H"
 
 #include "Evolve/WarpXDtType.H"
 #include "Evolve/WarpXPushType.H"
@@ -113,8 +114,7 @@ public:
         amrex::Real dt,
         DtType a_dt_type=DtType::Full,
         bool skip_deposition=false,
-        bool deposit_mass_matrices=false,
-        PushType push_type=PushType::Explicit
+        ImplicitOptions const * implicit_options = nullptr
     );
 
     /**

Source/Particles/MultiParticleContainer.cpp

@@ -455,7 +455,7 @@ MultiParticleContainer::Evolve (ablastr::fields::MultiFabRegister& fields,
                                 int lev,
                                 std::string const& current_fp_string,
                                 Real t, Real dt, DtType a_dt_type, bool skip_deposition,
-                                bool deposit_mass_matrices, PushType push_type)
+                                ImplicitOptions const * implicit_options)
 {
     if (! skip_deposition) {
         using ablastr::fields::Direction;
@@ -468,7 +468,7 @@ MultiParticleContainer::Evolve (ablastr::fields::MultiFabRegister& fields,
         if (fields.has(FieldType::current_buf, Direction{2}, lev)) { fields.get(FieldType::current_buf, Direction{2}, lev)->setVal(0.0); }
         if (fields.has(FieldType::rho_fp, lev)) { fields.get(FieldType::rho_fp, lev)->setVal(0.0); }
         if (fields.has(FieldType::rho_buf, lev)) { fields.get(FieldType::rho_buf, lev)->setVal(0.0); }
-        if (deposit_mass_matrices) {
+        if (implicit_options && implicit_options->deposit_mass_matrices) {
             fields.get(FieldType::MassMatrices_X, Direction{0}, lev)->setVal(0.0);
             fields.get(FieldType::MassMatrices_X, Direction{1}, lev)->setVal(0.0);
             fields.get(FieldType::MassMatrices_X, Direction{2}, lev)->setVal(0.0);
@@ -481,7 +481,7 @@ MultiParticleContainer::Evolve (ablastr::fields::MultiFabRegister& fields,
         }
     }
     for (auto& pc : allcontainers) {
-        pc->Evolve(fields, lev, current_fp_string, t, dt, a_dt_type, skip_deposition, deposit_mass_matrices, push_type);
+        pc->Evolve(fields, lev, current_fp_string, t, dt, a_dt_type, skip_deposition, implicit_options);
     }
 }