Skip to content

Commit 343dfd6

Browse files
authored
Fix FieldPoyntingFlux diagnostic for GPU (BLAST-WarpX#5780)
The FieldPoyntingFlux reduced diagnostic uses `amrex::ReduceOps` to calculate the diagnostic. The loops are structured so that the calculation is only done on the boundaries, by limiting the boxes to only include the boundary cells. For any blocks in the interior, the box will be empty. The `amrex::ReduceOps` on an empty box works Ok on the CPU but crashes on the GPU. The solution is to only call the reduction when the box is not empty.
1 parent 65db421 commit 343dfd6

File tree

1 file changed

+28
-25
lines changed

1 file changed

+28
-25
lines changed

Source/Diagnostics/ReducedDiags/FieldPoyntingFlux.cpp

Lines changed: 28 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -254,31 +254,34 @@ void FieldPoyntingFlux::ComputePoyntingFlux ()
254254
};
255255

256256
// Compute E x B
257-
reduce_ops.eval(box, reduce_data,
258-
[=] AMREX_GPU_DEVICE (int i, int j, int k) -> amrex::GpuTuple<amrex::Real>
259-
{
260-
amrex::Real Ex_cc = 0._rt, Ey_cc = 0._rt, Ez_cc = 0._rt;
261-
amrex::Real Bx_cc = 0._rt, By_cc = 0._rt, Bz_cc = 0._rt;
262-
263-
if (normal_dir == 1 || normal_dir == 2) {
264-
Ex_cc = ablastr::coarsen::sample::Interp(Ex_arr, Ex_stag, cc, cr, i, j, k, comp);
265-
Bx_cc = ablastr::coarsen::sample::Interp(Bx_arr, Bx_stag, cc, cr, i, j, k, comp);
266-
}
267-
268-
if (normal_dir == 0 || normal_dir == 2) {
269-
Ey_cc = ablastr::coarsen::sample::Interp(Ey_arr, Ey_stag, cc, cr, i, j, k, comp);
270-
By_cc = ablastr::coarsen::sample::Interp(By_arr, By_stag, cc, cr, i, j, k, comp);
271-
}
272-
if (normal_dir == 0 || normal_dir == 1) {
273-
Ez_cc = ablastr::coarsen::sample::Interp(Ez_arr, Ez_stag, cc, cr, i, j, k, comp);
274-
Bz_cc = ablastr::coarsen::sample::Interp(Bz_arr, Bz_stag, cc, cr, i, j, k, comp);
275-
}
276-
277-
amrex::Real const af = area_factor(i,j,k);
278-
if (normal_dir == 0) { return af*(Ey_cc * Bz_cc - Ez_cc * By_cc); }
279-
else if (normal_dir == 1) { return af*(Ez_cc * Bx_cc - Ex_cc * Bz_cc); }
280-
else { return af*(Ex_cc * By_cc - Ey_cc * Bx_cc); }
281-
});
257+
// On GPU, reduce_ops doesn't work with empty boxes.
258+
if (box.ok()) {
259+
reduce_ops.eval(box, reduce_data,
260+
[=] AMREX_GPU_DEVICE (int i, int j, int k) -> amrex::GpuTuple<amrex::Real>
261+
{
262+
amrex::Real Ex_cc = 0._rt, Ey_cc = 0._rt, Ez_cc = 0._rt;
263+
amrex::Real Bx_cc = 0._rt, By_cc = 0._rt, Bz_cc = 0._rt;
264+
265+
if (normal_dir == 1 || normal_dir == 2) {
266+
Ex_cc = ablastr::coarsen::sample::Interp(Ex_arr, Ex_stag, cc, cr, i, j, k, comp);
267+
Bx_cc = ablastr::coarsen::sample::Interp(Bx_arr, Bx_stag, cc, cr, i, j, k, comp);
268+
}
269+
270+
if (normal_dir == 0 || normal_dir == 2) {
271+
Ey_cc = ablastr::coarsen::sample::Interp(Ey_arr, Ey_stag, cc, cr, i, j, k, comp);
272+
By_cc = ablastr::coarsen::sample::Interp(By_arr, By_stag, cc, cr, i, j, k, comp);
273+
}
274+
if (normal_dir == 0 || normal_dir == 1) {
275+
Ez_cc = ablastr::coarsen::sample::Interp(Ez_arr, Ez_stag, cc, cr, i, j, k, comp);
276+
Bz_cc = ablastr::coarsen::sample::Interp(Bz_arr, Bz_stag, cc, cr, i, j, k, comp);
277+
}
278+
279+
amrex::Real const af = area_factor(i,j,k);
280+
if (normal_dir == 0) { return af*(Ey_cc * Bz_cc - Ez_cc * By_cc); }
281+
else if (normal_dir == 1) { return af*(Ez_cc * Bx_cc - Ex_cc * Bz_cc); }
282+
else { return af*(Ex_cc * By_cc - Ey_cc * Bx_cc); }
283+
});
284+
}
282285
}
283286

284287
int const sign = (face().isLow() ? -1 : 1);

0 commit comments

Comments
 (0)