Implement Checkpoint / Restart for Laser Particles (#2360)

* Initialize lasers when restarting + recalculate z position at each iteration

* Fix restart for rigid-injected particles

* Fix laser injection box at restart

* Use correct positions for the moving window of the BTD

* implement restart for laser particles

* add CI test for laser acceleration restarting

* writespace

* Revert "Fix restart for rigid-injected particles"

This reverts commit 33bf3a6.

* don't add lasers to all full diags, only checkpoints

* fix unused parameter warning

* move the call to reset the m_laser_injection_box into PostRestart

* bugfix for non-boosted frame simulations

* Read the mean velocity of the injected species

* Remove unused line

* Revert "Initialize lasers when restarting + recalculate z position at each iteration"

This reverts commit 322634c.

* Add new ReadHeader / WriteHeader functions

* Add WriteHeader/ReadHeader for laser particles

* Revert "add CI test for laser acceleration restarting"

This reverts commit 6ba0fb3.

* Fix merge commit

* Correct reading of updated position

* Add laser in restart test

* Remove unused parameters

* Update regression tests

* Add benchmark files

* Update boundary conditions for restart_psatd

Co-authored-by: Remi Lehe <[email protected]>

Author: atmyers

Examples/Tests/restart/inputs

@@ -28,7 +28,6 @@ warpx.cfl = .99
 warpx.do_moving_window = 1
 warpx.moving_window_dir = z
 warpx.moving_window_v = 1. # in units of the speed of light
-my_constants.lramp = 8.e-3
 my_constants.dens  = 1e+23
 
 # Order of particle shape factors
@@ -93,12 +92,14 @@ plasma_e.mass = m_e
 plasma_e.injection_style = "NUniformPerCell"
 plasma_e.zmin = -100.e-6
 plasma_e.zmax = 0.2
-plasma_e.xmin = -70.e-6
-plasma_e.xmax =  70.e-6
-plasma_e.ymin = -70.e-6
-plasma_e.ymax =  70.e-6
-plasma_e.profile = parse_density_function
-plasma_e.density_function(x,y,z) = "(z<lramp)*0.5*(1-cos(pi*z/lramp))*dens+(z>lramp)*dens"
+plasma_e.xmin = -100.e-6
+plasma_e.xmax =  100.e-6
+plasma_e.ymin = -100.e-6
+plasma_e.ymax =  100.e-6
+plasma_e.profile                 = "predefined"
+plasma_e.predefined_profile_name = "parabolic_channel"
+# predefined_profile_params = z_start   ramp_up plateau ramp_down rc  n0
+plasma_e.predefined_profile_params = 0. 0. 3. 0. 40.e-6 dens
 plasma_e.num_particles_per_cell_each_dim = 1 1 1
 plasma_e.momentum_distribution_type = "constant"
 plasma_e.ux = 0.0
@@ -111,12 +112,14 @@ plasma_p.mass = m_p
 plasma_p.injection_style = "NUniformPerCell"
 plasma_p.zmin = -100.e-6
 plasma_p.zmax = 0.2
-plasma_p.profile = parse_density_function
-plasma_p.density_function(x,y,z) = "(z<lramp)*0.5*(1-cos(pi*z/lramp))*dens+(z>lramp)*dens"
-plasma_p.xmin = -70.e-6
-plasma_p.xmax =  70.e-6
-plasma_p.ymin = -70.e-6
-plasma_p.ymax =  70.e-6
+plasma_p.profile                 = "predefined"
+plasma_p.predefined_profile_name = "parabolic_channel"
+# predefined_profile_params = z_start   ramp_up plateau ramp_down rc  n0
+plasma_p.predefined_profile_params = 0. 0. 3. 0. 40.e-6 dens
+plasma_p.xmin = -100.e-6
+plasma_p.xmax =  100.e-6
+plasma_p.ymin = -100.e-6
+plasma_p.ymax =  100.e-6
 plasma_p.num_particles_per_cell_each_dim = 1 1 1
 plasma_p.momentum_distribution_type = "constant"
 plasma_p.ux = 0.0
@@ -145,9 +148,21 @@ beam.uz_th = 200.
 beam.zinject_plane = .8e-3
 beam.rigid_advance = true
 
+lasers.names        = laser1
+laser1.profile      = Gaussian
+laser1.position     = 0. 0. 0.e-6 # This point is on the laser plane
+laser1.direction    = 0. 0. 1.     # The plane normal direction
+laser1.polarization = 1. 0. 0.    # The main polarization vector
+laser1.e_max        = 16.e12        # Maximum amplitude of the laser field (in V/m)
+laser1.profile_waist = 40.e-6      # The waist of the laser (in meters)
+laser1.profile_duration = 30.e-15  # The duration of the laser (in seconds)
+laser1.profile_t_peak = 60.e-15    # The time at which the laser reaches its peak (in seconds)
+laser1.profile_focal_distance = 0.e-6  # Focal distance from the antenna (in meters)
+laser1.wavelength = 5.e-6         # The wavelength of the laser (in meters)
+
 # Diagnostics
 diagnostics.diags_names = diag1 chk
-diag1.intervals = 1
+diag1.intervals = 5
 diag1.diag_type = Full
 chk.intervals = 5
 chk.diag_type = Full

Regression/Checksum/benchmarks_json/restart.json

@@ -33,36 +33,36 @@
     "particle_weight": 6241509074.460762
   },
   "lev=0": {
-    "Bx": 98192.19947145433,
-    "By": 98202.24044988544,
-    "Bz": 32.176883437432316,
-    "Ex": 14065714408245.07,
-    "Ey": 14064073072398.174,
-    "Ez": 33307954593489.707,
-    "jx": 304667056543.9146,
-    "jy": 180701392073.9691,
-    "jz": 904275753595636.4
+    "Bx": 115327.15140818382,
+    "By": 242416.19187313571,
+    "Bz": 62096.66973827369,
+    "Ex": 120130722555567.77,
+    "Ey": 20867597109617.934,
+    "Ez": 53551348241648.52,
+    "jx": 2.1551699509760676e+16,
+    "jy": 332122730528260.0,
+    "jz": 4524888666510641.0
   },
   "plasma_e": {
-    "particle_cpu": 4116.0,
-    "particle_id": 22919946.0,
-    "particle_momentum_x": 5.40817681171179e-22,
-    "particle_momentum_y": 5.385133314763837e-22,
-    "particle_momentum_z": 1.1184395632063724e-17,
-    "particle_position_x": 0.1350593467304257,
-    "particle_position_y": 0.1350593241232737,
-    "particle_position_z": 0.20362721013846566,
-    "particle_weight": 126912632943.3621
+    "particle_cpu": 9680.0,
+    "particle_id": 117349672.0,
+    "particle_momentum_x": 2.6417827387235982e-19,
+    "particle_momentum_y": 1.3246227891556811e-20,
+    "particle_momentum_z": 2.632691328351647e-17,
+    "particle_position_x": 0.49916178543521594,
+    "particle_position_y": 0.4991848022446371,
+    "particle_position_z": 0.45626376648502054,
+    "particle_weight": 33067341227104.594
   },
   "plasma_p": {
-    "particle_cpu": 4116.0,
-    "particle_id": 25328394.0,
-    "particle_momentum_x": 5.408310433658191e-22,
-    "particle_momentum_y": 5.385274435809097e-22,
-    "particle_momentum_z": 2.0535791481240203e-14,
-    "particle_position_x": 0.13505624831336835,
-    "particle_position_y": 0.13505624832567958,
-    "particle_position_z": 0.2036271949467523,
-    "particle_weight": 126912632943.3621
+    "particle_cpu": 9680.0,
+    "particle_id": 128125448.0,
+    "particle_momentum_x": 2.6389034754489564e-19,
+    "particle_momentum_y": 5.3122104782787265e-21,
+    "particle_momentum_z": 4.8296005971893094e-14,
+    "particle_position_x": 0.4991250026192299,
+    "particle_position_y": 0.4991249980376633,
+    "particle_position_z": 0.4563845472659988,
+    "particle_weight": 33067341227104.594
   }
 }

Regression/Checksum/benchmarks_json/restart_psatd.json

@@ -33,36 +33,36 @@
     "particle_weight": 6241509074.460762
   },
   "lev=0": {
-    "Bx": 97806.19840108546,
-    "By": 97815.85756771098,
-    "Bz": 31.732427701722912,
-    "Ex": 12877754924119.082,
-    "Ey": 12876901684280.229,
-    "Ez": 32448463177834.543,
-    "jx": 307666015585.9687,
-    "jy": 182363554739.9786,
-    "jz": 904274413323325.2
+    "Bx": 114349.71019313535,
+    "By": 237246.30659111793,
+    "Bz": 61241.57839321267,
+    "Ex": 113310667309569.25,
+    "Ey": 18419343979199.57,
+    "Ez": 50966992135551.99,
+    "jx": 2.343755417758882e+16,
+    "jy": 264692970881696.72,
+    "jz": 4106459950194407.0
   },
   "plasma_e": {
-    "particle_cpu": 4116.0,
-    "particle_id": 22919946.0,
-    "particle_momentum_x": 5.685881126709523e-22,
-    "particle_momentum_y": 5.6629711844429285e-22,
-    "particle_momentum_z": 1.1184398018026139e-17,
-    "particle_position_x": 0.1350592433885383,
-    "particle_position_y": 0.1350592219411637,
-    "particle_position_z": 0.20362720921987482,
-    "particle_weight": 126912632943.36244
+    "particle_cpu": 9680.0,
+    "particle_id": 117349672.0,
+    "particle_momentum_x": 1.473920377077317e-19,
+    "particle_momentum_y": 1.0295357463154818e-20,
+    "particle_momentum_z": 2.631936992393973e-17,
+    "particle_position_x": 0.4991563943788139,
+    "particle_position_y": 0.49917524942325764,
+    "particle_position_z": 0.4562708055573991,
+    "particle_weight": 33067341227104.625
   },
   "plasma_p": {
-    "particle_cpu": 4116.0,
-    "particle_id": 25328394.0,
-    "particle_momentum_x": 5.686071969908394e-22,
-    "particle_momentum_y": 5.663169754132697e-22,
-    "particle_momentum_z": 2.0535791478848e-14,
-    "particle_position_x": 0.13505624836966013,
-    "particle_position_y": 0.13505624838133984,
-    "particle_position_z": 0.20362719494726383,
-    "particle_weight": 126912632943.36244
+    "particle_cpu": 9680.0,
+    "particle_id": 128125448.0,
+    "particle_momentum_x": 1.4773149979258644e-19,
+    "particle_momentum_y": 3.3342009095840554e-21,
+    "particle_momentum_z": 4.8296010015063954e-14,
+    "particle_position_x": 0.4991250009706611,
+    "particle_position_y": 0.49912499871191435,
+    "particle_position_z": 0.4563845474003356,
+    "particle_weight": 33067341227104.625
   }
 }

Regression/Checksum/benchmarks_json/restart_psatd_time_avg.json

@@ -33,36 +33,36 @@
     "particle_weight": 6241509074.460762
   },
   "lev=0": {
-    "Bx": 97799.68920761721,
-    "By": 97809.22760976445,
-    "Bz": 31.138421896287632,
-    "Ex": 12848024095028.355,
-    "Ey": 12847124613971.557,
-    "Ez": 32437971780449.27,
-    "jx": 308194378521.1045,
-    "jy": 182890492089.67697,
-    "jz": 904274843199463.8
+    "Bx": 113572.32066454006,
+    "By": 233769.68176084137,
+    "Bz": 58935.90471405687,
+    "Ex": 111077830627827.34,
+    "Ey": 18104292500486.71,
+    "Ez": 49957933531975.0,
+    "jx": 2.3032421193292172e+16,
+    "jy": 252044670800869.56,
+    "jz": 4052661521670030.0
   },
   "plasma_e": {
-    "particle_cpu": 4116.0,
-    "particle_id": 22919946.0,
-    "particle_momentum_x": 5.744112487812186e-22,
-    "particle_momentum_y": 5.721125891605127e-22,
-    "particle_momentum_z": 1.1184397774190402e-17,
-    "particle_position_x": 0.13505927138536114,
-    "particle_position_y": 0.13505924998293056,
-    "particle_position_z": 0.20362720922167785,
-    "particle_weight": 126912632943.36244
+    "particle_cpu": 9680.0,
+    "particle_id": 117349672.0,
+    "particle_momentum_x": 1.3889342284437047e-19,
+    "particle_momentum_y": 9.709966613374186e-21,
+    "particle_momentum_z": 2.6318578216093305e-17,
+    "particle_position_x": 0.4991547138014627,
+    "particle_position_y": 0.49917221491130515,
+    "particle_position_z": 0.4562788688974234,
+    "particle_weight": 33067341227104.625
   },
   "plasma_p": {
-    "particle_cpu": 4116.0,
-    "particle_id": 25328394.0,
-    "particle_momentum_x": 5.744307393038055e-22,
-    "particle_momentum_y": 5.721328534468911e-22,
-    "particle_momentum_z": 2.0535791479091845e-14,
-    "particle_position_x": 0.13505624835441166,
-    "particle_position_y": 0.13505624836606694,
-    "particle_position_z": 0.20362719494726217,
-    "particle_weight": 126912632943.36244
+    "particle_cpu": 9680.0,
+    "particle_id": 128125448.0,
+    "particle_momentum_x": 1.3957368399934083e-19,
+    "particle_momentum_y": 3.3829656839017584e-21,
+    "particle_momentum_z": 4.829601122286012e-14,
+    "particle_position_x": 0.4991250008410111,
+    "particle_position_y": 0.4991249984371928,
+    "particle_position_z": 0.4563845474251499,
+    "particle_weight": 33067341227104.625
   }
 }

Regression/WarpX-tests.ini

@@ -912,7 +912,7 @@ tolerance = 1.e-14
 [restart_psatd]
 buildDir = .
 inputFile = Examples/Tests/restart/inputs
-runtime_params = algo.maxwell_solver=psatd psatd.use_default_v_galilean=1 particles.use_fdtd_nci_corr=0 chk.file_prefix=restart_psatd_chk
+runtime_params = algo.maxwell_solver=psatd psatd.use_default_v_galilean=1 particles.use_fdtd_nci_corr=0 chk.file_prefix=restart_psatd_chk boundary.field_lo=periodic periodic damped boundary.field_hi=periodic periodic damped
 dim = 3
 addToCompileString = USE_PSATD=TRUE
 restartTest = 1

Source/Diagnostics/FullDiagnostics.cpp

@@ -54,7 +54,13 @@ FullDiagnostics::InitializeParticleBuffer ()
 
     const MultiParticleContainer& mpc = warpx.GetPartContainer();
     // If not specified, dump all species
-    if (m_output_species_names.empty()) m_output_species_names = mpc.GetSpeciesNames();
+    if (m_output_species_names.empty()) {
+        if (m_format == "checkpoint") {
+            m_output_species_names = mpc.GetSpeciesAndLasersNames();
+        } else {
+            m_output_species_names = mpc.GetSpeciesNames();
+        }
+    }
     // Initialize one ParticleDiag per species requested
     for (auto const& species : m_output_species_names){
         const int idx = mpc.getSpeciesID(species);