Skip to content

Commit 0f7796b

Browse files
RemiLeheRemiLehe
authored
Merge pull request #107 from ECP-WarpX/runtime_boosted_frame_particles
Runtime boosted frame particles
2 parents e80b825 + deea0e1 commit 0f7796b

File tree

11 files changed

+182
-95
lines changed

11 files changed

+182
-95
lines changed

‎GNUmakefile‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,6 @@ TINY_PROFILE = TRUE
1717
#COMM_PROFILE = TRUE
1818
#TRACE_PROFILE = TRUE
1919

20-
STORE_OLD_PARTICLE_ATTRIBS = FALSE
21-
2220
USE_OMP = TRUE
2321
USE_GPU = FALSE
2422

‎Source/Diagnostics/WarpXIO.cpp‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -631,15 +631,16 @@ WarpX::WritePlotFile () const
631631
particle_varnames.push_back("theta");
632632
#endif
633633

634-
#ifdef WARPX_STORE_OLD_PARTICLE_ATTRIBS
635-
particle_varnames.push_back("xold");
636-
particle_varnames.push_back("yold");
637-
particle_varnames.push_back("zold");
638-
639-
particle_varnames.push_back("uxold");
640-
particle_varnames.push_back("uyold");
641-
particle_varnames.push_back("uzold");
642-
#endif
634+
if (WarpX::do_boosted_frame_diagnostic && WarpX::do_boosted_frame_particles)
635+
{
636+
particle_varnames.push_back("xold");
637+
particle_varnames.push_back("yold");
638+
particle_varnames.push_back("zold");
639+
640+
particle_varnames.push_back("uxold");
641+
particle_varnames.push_back("uyold");
642+
particle_varnames.push_back("uzold");
643+
}
643644

644645
mypc->WritePlotFile(plotfilename, particle_plot_flags, particle_varnames);
645646

‎Source/FortranInterface/WarpX_f.H‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -81,17 +81,16 @@ extern "C"
8181
{
8282
#endif
8383

84-
#ifdef WARPX_STORE_OLD_PARTICLE_ATTRIBS
8584
void warpx_copy_attribs(const long* np,
8685
const amrex_real* xp, const amrex_real* yp, const amrex_real* zp,
8786
const amrex_real* uxp, const amrex_real* uyp, const amrex_real* uzp,
8887
amrex_real* xpold, amrex_real* ypold, amrex_real* zpold,
8988
amrex_real* uxpold, amrex_real* uypold, amrex_real* uzpold);
90-
#endif
91-
void WRPX_COPY_SLICE(const int* lo, const int* hi,
92-
const amrex_real* tmp, const int* tlo, const int* thi,
93-
amrex_real* buf, const int* blo, const int* bhi,
94-
const int* ncomp, const int* i_boost, const int* i_lab);
89+
90+
void WRPX_COPY_SLICE(const int* lo, const int* hi,
91+
const amrex_real* tmp, const int* tlo, const int* thi,
92+
amrex_real* buf, const int* blo, const int* bhi,
93+
const int* ncomp, const int* i_boost, const int* i_lab);
9594

9695
// Charge deposition
9796
void warpx_charge_deposition(amrex::Real* rho,

‎Source/Make.WarpX‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -123,10 +123,6 @@ ifeq ($(USE_RZ),TRUE)
123123
DEFINES += -DWARPX_RZ
124124
endif
125125

126-
ifeq ($(STORE_OLD_PARTICLE_ATTRIBS),TRUE)
127-
DEFINES += -DWARPX_STORE_OLD_PARTICLE_ATTRIBS
128-
endif
129-
130126
ifeq ($(DO_ELECTROSTATIC),TRUE)
131127
include $(AMREX_HOME)/Src/LinearSolvers/C_to_F_MG/Make.package
132128
include $(AMREX_HOME)/Src/LinearSolvers/F_MG/FParallelMG.mak

‎Source/Particles/MultiParticleContainer.cpp‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,26 @@ MultiParticleContainer::MultiParticleContainer (AmrCore* amr_core)
2929
allcontainers[i].reset(new LaserParticleContainer(amr_core,i, lasers_names[i-nspecies]));
3030
}
3131

32-
pc_tmp.reset(new PhysicalParticleContainer(amr_core));
32+
pc_tmp.reset(new PhysicalParticleContainer(amr_core));
33+
34+
if (WarpX::do_boosted_frame_diagnostic && WarpX::do_boosted_frame_particles)
35+
{
36+
for (int i = 0; i < nspecies + nlasers; ++i)
37+
{
38+
allcontainers[i]->AddRealComp("xold");
39+
allcontainers[i]->AddRealComp("yold");
40+
allcontainers[i]->AddRealComp("zold");
41+
allcontainers[i]->AddRealComp("uxold");
42+
allcontainers[i]->AddRealComp("uyold");
43+
allcontainers[i]->AddRealComp("uzold");
44+
}
45+
pc_tmp->AddRealComp("xold");
46+
pc_tmp->AddRealComp("yold");
47+
pc_tmp->AddRealComp("zold");
48+
pc_tmp->AddRealComp("uxold");
49+
pc_tmp->AddRealComp("uyold");
50+
pc_tmp->AddRealComp("uzold");
51+
}
3352
}
3453

3554
void

‎Source/Particles/PhysicalParticleContainer.cpp‎

Lines changed: 56 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -184,6 +184,18 @@ PhysicalParticleContainer::AddGaussianBeam(Real x_m, Real y_m, Real z_m,
184184
attribs[PIdx::uz] = u[2];
185185
attribs[PIdx::w ] = weight;
186186

187+
if (WarpX::do_boosted_frame_diagnostic && WarpX::do_boosted_frame_particles)
188+
{
189+
auto& particle_tile = DefineAndReturnParticleTile(0, 0, 0);
190+
particle_tile.push_back_real(particle_comps["xold"], x);
191+
particle_tile.push_back_real(particle_comps["yold"], y);
192+
particle_tile.push_back_real(particle_comps["zold"], z);
193+
194+
particle_tile.push_back_real(particle_comps["uxold"], u[0]);
195+
particle_tile.push_back_real(particle_comps["uyold"], u[1]);
196+
particle_tile.push_back_real(particle_comps["uzold"], u[2]);
197+
}
198+
187199
AddOneParticle(0, 0, 0, x, y, z, attribs);
188200
}
189201
}
@@ -455,16 +467,18 @@ PhysicalParticleContainer::AddPlasmaCPU (int lev, RealBox part_realbox)
455467
attribs[PIdx::ux] = u[0];
456468
attribs[PIdx::uy] = u[1];
457469
attribs[PIdx::uz] = u[2];
458-
459-
#ifdef WARPX_STORE_OLD_PARTICLE_ATTRIBS
460-
attribs[PIdx::xold] = x;
461-
attribs[PIdx::yold] = y;
462-
attribs[PIdx::zold] = z;
463-
464-
attribs[PIdx::uxold] = u[0];
465-
attribs[PIdx::uyold] = u[1];
466-
attribs[PIdx::uzold] = u[2];
467-
#endif
470+
471+
if (WarpX::do_boosted_frame_diagnostic && WarpX::do_boosted_frame_particles)
472+
{
473+
auto& particle_tile = DefineAndReturnParticleTile(lev, grid_id, tile_id);
474+
particle_tile.push_back_real(particle_comps["xold"], x);
475+
particle_tile.push_back_real(particle_comps["yold"], y);
476+
particle_tile.push_back_real(particle_comps["zold"], z);
477+
478+
particle_tile.push_back_real(particle_comps["uxold"], u[0]);
479+
particle_tile.push_back_real(particle_comps["uyold"], u[1]);
480+
particle_tile.push_back_real(particle_comps["uzold"], u[2]);
481+
}
468482

469483
AddOneParticle(lev, grid_id, tile_id, x, y, z, attribs);
470484
}
@@ -695,15 +709,18 @@ PhysicalParticleContainer::AddPlasmaGPU (int lev, RealBox part_realbox)
695709
attribs[PIdx::uy] = u[1];
696710
attribs[PIdx::uz] = u[2];
697711

698-
#ifdef WARPX_STORE_OLD_PARTICLE_ATTRIBS
699-
attribs[PIdx::xold] = x;
700-
attribs[PIdx::yold] = y;
701-
attribs[PIdx::zold] = z;
702-
703-
attribs[PIdx::uxold] = u[0];
704-
attribs[PIdx::uyold] = u[1];
705-
attribs[PIdx::uzold] = u[2];
706-
#endif
712+
// note - this will be slow on the GPU, need to revisit
713+
if (WarpX::do_boosted_frame_diagnostic && WarpX::do_boosted_frame_particles)
714+
{
715+
auto& particle_tile = DefineAndReturnParticleTile(lev, grid_id, tile_id);
716+
particle_tile.push_back_real(particle_comps["xold"], x);
717+
particle_tile.push_back_real(particle_comps["yold"], y);
718+
particle_tile.push_back_real(particle_comps["zold"], z);
719+
720+
particle_tile.push_back_real(particle_comps["uxold"], u[0]);
721+
particle_tile.push_back_real(particle_comps["uyold"], u[1]);
722+
particle_tile.push_back_real(particle_comps["uzold"], u[2]);
723+
}
707724

708725
ParticleType p;
709726
p.id() = ParticleType::NextID();
@@ -1666,20 +1683,20 @@ PhysicalParticleContainer::PushPX(WarpXParIter& pti,
16661683
auto& Bzp = attribs[PIdx::Bz];
16671684
const long np = pti.numParticles();
16681685

1669-
#ifdef WARPX_STORE_OLD_PARTICLE_ATTRIBS
1670-
auto& xpold = attribs[PIdx::xold];
1671-
auto& ypold = attribs[PIdx::yold];
1672-
auto& zpold = attribs[PIdx::zold];
1673-
auto& uxpold = attribs[PIdx::uxold];
1674-
auto& uypold = attribs[PIdx::uyold];
1675-
auto& uzpold = attribs[PIdx::uzold];
1676-
1677-
warpx_copy_attribs(&np, xp.dataPtr(), yp.dataPtr(), zp.dataPtr(),
1678-
uxp.dataPtr(), uyp.dataPtr(), uzp.dataPtr(),
1679-
xpold.dataPtr(), ypold.dataPtr(), zpold.dataPtr(),
1680-
uxpold.dataPtr(), uypold.dataPtr(), uzpold.dataPtr());
1681-
1682-
#endif
1686+
if (WarpX::do_boosted_frame_diagnostic && WarpX::do_boosted_frame_particles)
1687+
{
1688+
auto& xpold = pti.GetAttribs(particle_comps["xold"]);
1689+
auto& ypold = pti.GetAttribs(particle_comps["yold"]);
1690+
auto& zpold = pti.GetAttribs(particle_comps["zold"]);
1691+
auto& uxpold = pti.GetAttribs(particle_comps["uxold"]);
1692+
auto& uypold = pti.GetAttribs(particle_comps["uyold"]);
1693+
auto& uzpold = pti.GetAttribs(particle_comps["uzold"]);
1694+
1695+
warpx_copy_attribs(&np, xp.dataPtr(), yp.dataPtr(), zp.dataPtr(),
1696+
uxp.dataPtr(), uyp.dataPtr(), uzp.dataPtr(),
1697+
xpold.dataPtr(), ypold.dataPtr(), zpold.dataPtr(),
1698+
uxpold.dataPtr(), uypold.dataPtr(), uzpold.dataPtr());
1699+
}
16831700

16841701
warpx_particle_pusher(&np,
16851702
xp.dataPtr(),
@@ -1801,7 +1818,6 @@ void PhysicalParticleContainer::GetParticleSlice(const int direction, const Real
18011818
{
18021819
BL_PROFILE("PhysicalParticleContainer::GetParticleSlice");
18031820

1804-
#ifdef WARPX_STORE_OLD_PARTICLE_ATTRIBS
18051821
// Assume that the boost in the positive z direction.
18061822
#if (AMREX_SPACEDIM == 2)
18071823
AMREX_ALWAYS_ASSERT(direction == 1);
@@ -1864,12 +1880,12 @@ void PhysicalParticleContainer::GetParticleSlice(const int direction, const Real
18641880
auto& uyp_new = attribs[PIdx::uy ];
18651881
auto& uzp_new = attribs[PIdx::uz ];
18661882

1867-
auto& xp_old = attribs[PIdx::xold ];
1868-
auto& yp_old = attribs[PIdx::yold ];
1869-
auto& zp_old = attribs[PIdx::zold ];
1870-
auto& uxp_old = attribs[PIdx::uxold];
1871-
auto& uyp_old = attribs[PIdx::uyold];
1872-
auto& uzp_old = attribs[PIdx::uzold];
1883+
auto& xp_old = pti.GetAttribs(particle_comps["xold"]);
1884+
auto& yp_old = pti.GetAttribs(particle_comps["yold"]);
1885+
auto& zp_old = pti.GetAttribs(particle_comps["zold"]);
1886+
auto& uxp_old = pti.GetAttribs(particle_comps["uxold"]);
1887+
auto& uyp_old = pti.GetAttribs(particle_comps["uyold"]);
1888+
auto& uzp_old = pti.GetAttribs(particle_comps["uzold"]);
18731889

18741890
const long np = pti.numParticles();
18751891

@@ -1919,10 +1935,6 @@ void PhysicalParticleContainer::GetParticleSlice(const int direction, const Real
19191935
}
19201936
}
19211937
}
1922-
#else
1923-
AMREX_ALWAYS_ASSERT_WITH_MESSAGE( false ,
1924-
"ERROR: WarpX must be compiled with STORE_OLD_PARTICLE_ATTRIBS=TRUE to use the back-transformed diagnostics");
1925-
#endif
19261938
}
19271939

19281940
int PhysicalParticleContainer::GetRefineFac(const Real x, const Real y, const Real z)

‎Source/Particles/RigidInjectedParticleContainer.cpp‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -225,20 +225,20 @@ RigidInjectedParticleContainer::PushPX(WarpXParIter& pti,
225225
auto& Bzp = attribs[PIdx::Bz];
226226
const long np = pti.numParticles();
227227

228-
#ifdef WARPX_STORE_OLD_PARTICLE_ATTRIBS
229-
auto& xpold = attribs[PIdx::xold];
230-
auto& ypold = attribs[PIdx::yold];
231-
auto& zpold = attribs[PIdx::zold];
232-
auto& uxpold = attribs[PIdx::uxold];
233-
auto& uypold = attribs[PIdx::uyold];
234-
auto& uzpold = attribs[PIdx::uzold];
235-
236-
warpx_copy_attribs(&np, xp.dataPtr(), yp.dataPtr(), zp.dataPtr(),
237-
uxp.dataPtr(), uyp.dataPtr(), uzp.dataPtr(),
238-
xpold.dataPtr(), ypold.dataPtr(), zpold.dataPtr(),
239-
uxpold.dataPtr(), uypold.dataPtr(), uzpold.dataPtr());
240-
241-
#endif
228+
if (WarpX::do_boosted_frame_diagnostic && WarpX::do_boosted_frame_particles)
229+
{
230+
auto& xpold = pti.GetAttribs(particle_comps["xold"]);
231+
auto& ypold = pti.GetAttribs(particle_comps["yold"]);
232+
auto& zpold = pti.GetAttribs(particle_comps["zold"]);
233+
auto& uxpold = pti.GetAttribs(particle_comps["uxold"]);
234+
auto& uypold = pti.GetAttribs(particle_comps["uyold"]);
235+
auto& uzpold = pti.GetAttribs(particle_comps["uzold"]);
236+
237+
warpx_copy_attribs(&np, xp.dataPtr(), yp.dataPtr(), zp.dataPtr(),
238+
uxp.dataPtr(), uyp.dataPtr(), uzp.dataPtr(),
239+
xpold.dataPtr(), ypold.dataPtr(), zpold.dataPtr(),
240+
uxpold.dataPtr(), uypold.dataPtr(), uzpold.dataPtr());
241+
}
242242

243243
// Save the position and momenta, making copies
244244
Cuda::ManagedDeviceVector<Real> xp_save, yp_save, zp_save;

‎Source/Particles/WarpXParticleContainer.H‎

Lines changed: 18 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,6 @@ struct PIdx
1414
#ifdef WARPX_RZ
1515
theta, // RZ needs all three position components
1616
#endif
17-
#ifdef WARPX_STORE_OLD_PARTICLE_ATTRIBS
18-
xold, yold, zold, uxold, uyold, uzold,
19-
#endif
2017
nattribs
2118
};
2219
};
@@ -42,15 +39,7 @@ namespace ParticleStringNames
4239
{"Ez", PIdx::Ez },
4340
{"Bx", PIdx::Bx },
4441
{"By", PIdx::By },
45-
{"Bz", PIdx::Bz },
46-
#ifdef WARPX_STORE_OLD_PARTICLE_ATTRIBS
47-
{"xold", PIdx::xold },
48-
{"yold", PIdx::yold },
49-
{"zold", PIdx::zold },
50-
{"uxold", PIdx::uxold},
51-
{"uyold", PIdx::uyold},
52-
{"uzold", PIdx::uzold},
53-
#endif
42+
{"Bz", PIdx::Bz }
5443
};
5544
}
5645

@@ -231,8 +220,25 @@ public:
231220
// split along axes (0) or diagonals (1)
232221
int split_type = 0;
233222

223+
using amrex::ParticleContainer<0, 0, PIdx::nattribs>::AddRealComp;
224+
using amrex::ParticleContainer<0, 0, PIdx::nattribs>::AddIntComp;
225+
226+
void AddRealComp (const std::string& name, bool comm=true)
227+
{
228+
particle_comps[name] = NumRealComps();
229+
AddRealComp(comm);
230+
}
231+
232+
void AddIntComp (const std::string& name, bool comm=true)
233+
{
234+
particle_comps[name] = NumIntComps();
235+
AddIntComp(comm);
236+
}
237+
234238
protected:
235239

240+
std::map<std::string, int> particle_comps;
241+
236242
int species_id;
237243

238244
amrex::Real charge;

‎Source/Particles/WarpXParticleContainer.cpp‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,32 @@ WarpXParticleContainer::WarpXParticleContainer (AmrCore* amr_core, int ispecies)
6363
SetParticleSize();
6464
ReadParameters();
6565

66+
// build up the map of string names to particle component numbers
67+
particle_comps["w"] = PIdx::w;
68+
particle_comps["ux"] = PIdx::ux;
69+
particle_comps["uy"] = PIdx::uy;
70+
particle_comps["uz"] = PIdx::uz;
71+
particle_comps["Ex"] = PIdx::Ex;
72+
particle_comps["Ey"] = PIdx::Ey;
73+
particle_comps["Ez"] = PIdx::Ez;
74+
particle_comps["Bx"] = PIdx::Bx;
75+
particle_comps["By"] = PIdx::By;
76+
particle_comps["Bz"] = PIdx::Bz;
77+
#ifdef WARPX_RZ
78+
particle_comps["theta"] = PIdx::theta;
79+
#endif
80+
81+
if (WarpX::do_boosted_frame_diagnostic && WarpX::do_boosted_frame_particles)
82+
{
83+
particle_comps["xold"] = PIdx::nattribs;
84+
particle_comps["yold"] = PIdx::nattribs+1;
85+
particle_comps["zold"] = PIdx::nattribs+2;
86+
particle_comps["uxold"] = PIdx::nattribs+3;
87+
particle_comps["uyold"] = PIdx::nattribs+4;
88+
particle_comps["uzold"] = PIdx::nattribs+5;
89+
90+
}
91+
6692
// Initialize temporary local arrays for charge/current deposition
6793
int num_threads = 1;
6894
#ifdef _OPENMP
@@ -204,6 +230,15 @@ WarpXParticleContainer::AddNParticles (int lev,
204230
#endif
205231
p.pos(1) = z[i];
206232
#endif
233+
234+
if (WarpX::do_boosted_frame_diagnostic && WarpX::do_boosted_frame_particles)
235+
{
236+
auto& particle_tile = DefineAndReturnParticleTile(0, 0, 0);
237+
particle_tile.push_back_real(particle_comps["xold"], x[i]);
238+
particle_tile.push_back_real(particle_comps["yold"], y[i]);
239+
particle_tile.push_back_real(particle_comps["zold"], z[i]);
240+
}
241+
207242
particle_tile.push_back(p);
208243
}
209244

@@ -214,6 +249,14 @@ WarpXParticleContainer::AddNParticles (int lev,
214249
particle_tile.push_back_real(PIdx::uy, vy + ibegin, vy + iend);
215250
particle_tile.push_back_real(PIdx::uz, vz + ibegin, vz + iend);
216251

252+
if (WarpX::do_boosted_frame_diagnostic && WarpX::do_boosted_frame_particles)
253+
{
254+
auto& particle_tile = DefineAndReturnParticleTile(0, 0, 0);
255+
particle_tile.push_back_real(particle_comps["uxold"], vx + ibegin, vx + iend);
256+
particle_tile.push_back_real(particle_comps["uyold"], vy + ibegin, vy + iend);
257+
particle_tile.push_back_real(particle_comps["uzold"], vz + ibegin, vz + iend);
258+
}
259+
217260
for (int comp = PIdx::uz+1; comp < PIdx::nattribs; ++comp)
218261
{
219262
#ifdef WARPX_RZ

0 commit comments

Comments
 (0)