try initial suggestions for btd output for each level

Source/Diagnostics/BTDiagnostics.H

@@ -187,10 +187,10 @@ private:
     amrex::Vector<amrex::IntVect> m_snapshot_ncells_lab;
     /** Vector of Box-dimension in boosted-frame index space
      *  for each back-transformed snapshot */
-    amrex::Vector<amrex::Box> m_snapshot_box;
+    amrex::Vector< amrex::Vector <amrex::Box> > m_snapshot_box;
     /** Vector of Box-dimension in boosted-frame index space corresponding to the
      *  buffer that covers a part of the full backtransformed snapshot */
-    amrex::Vector<amrex::Box> m_buffer_box;
+    amrex::Vector< amrex::Vector <amrex::Box> > m_buffer_box;
     /** Vector of lab-frame z co-ordinate of each back-transformed snapshot
      *  at the current timestep */
     amrex::Vector<amrex::Real> m_current_z_lab;
@@ -238,7 +238,7 @@ private:
     /** Vector of k-indices in the hi-end along the moving window direction for the buffer being filled
      *  for each snapshot.
      */
-    amrex::Vector<int> m_buffer_k_index_hi;
+    amrex::Vector< amrex::Vector<int> > m_buffer_k_index_hi;
     /** Multi-level cell-centered multifab with all field-data components, namely,
      *  Ex, Ey, Ez, Bx, By, Bz, jx, jy, jz, and rho.
      *  This cell-centered data extending over the entire domain
@@ -307,8 +307,8 @@ private:
      * returns bool = true is buffer is full, that is,
                when buffer counter is equal to m_buffer_size
      */
-    bool buffer_full (int i_buffer) {
-        return (k_index_zlab(i_buffer,0) == m_buffer_box[i_buffer].smallEnd(m_moving_window_dir));
+    bool buffer_full (int i_buffer, const int lev) {
+        return (k_index_zlab(i_buffer,0) == m_buffer_box[i_buffer][lev].smallEnd(m_moving_window_dir));
     }
 
     /** whether field buffer is empty.
@@ -412,8 +412,8 @@ private:
 
     amrex::Real gettlab (int i_buffer) override {return m_t_lab[i_buffer];}
     void settlab (int i_buffer, amrex::Real tlab) override {m_t_lab[i_buffer] = tlab; }
-    int get_buffer_k_index_hi (int i_buffer) override {return m_buffer_k_index_hi[i_buffer]; }
-    void set_buffer_k_index_hi (int i_buffer, int kindex) override {m_buffer_k_index_hi[i_buffer] = kindex;}
+    int get_buffer_k_index_hi (int i_buffer, const int lev) override {return m_buffer_k_index_hi[i_buffer][lev]; }
+    void set_buffer_k_index_hi (int i_buffer, int kindex, const int lev) override {m_buffer_k_index_hi[i_buffer][lev] = kindex;}
     amrex::Real get_snapshot_domain_lo (int i_buffer, int idim) override {return m_snapshot_domain_lab[i_buffer].lo(idim); }
     amrex::Real get_snapshot_domain_hi (int i_buffer, int idim) override {return m_snapshot_domain_lab[i_buffer].hi(idim); }
     int get_flush_counter (int i_buffer) override {return m_buffer_flush_counter[i_buffer]; }

Source/Diagnostics/BoundaryScrapingDiagnostics.cpp

@@ -151,16 +151,17 @@ BoundaryScrapingDiagnostics::Flush (int i_buffer)
     int const numBTDBuffers = 0;
     // The data being written out is saved in a pinned particle container
     bool const use_pinned_pc = true;
-    const amrex::Geometry& geom = warpx.Geom(0); // For compatibility with `WriteToFile` ; not used
+    //const amrex::Vector<amrex::Geometry>& geom = warpx.Geom(0); // For compatibility with `WriteToFile` ; not used
+    amrex::Vector<amrex::Geometry> geom; // For compatibility with `WriteToFile` ; not used
 
     // The data for each boundary is written out to a separate directory with the boundary name
     const std::string file_prefix = m_file_prefix + "/particles_at_" + particle_buffer.boundaryName(i_buffer);
 
     m_flush_format->WriteToFile(
         m_varnames, m_mf_output[i_buffer], m_geom_output[i_buffer], warpx.getistep(),
         warpx.gett_new(0), m_output_species[i_buffer], nlev_output, file_prefix,
-        m_file_min_digits, false, false, use_pinned_pc, isBTD,
-        warpx.getistep(0), bufferID, numBTDBuffers, geom,
+        m_file_min_digits, false, false, geom, use_pinned_pc, isBTD,
+        warpx.getistep(0), bufferID, numBTDBuffers,
         isLastBTD, m_totalParticles_flushed_already[i_buffer]);
 
     // Now that the data has been written out, clear out the buffer

Source/Diagnostics/Diagnostics.H

@@ -120,12 +120,12 @@ public:
     /** Returns k-index in big end of the buffer box currently being filled for the ith snapshot in BTD
      * \param[in] i_buffer index of the buffer
      */
-    virtual int get_buffer_k_index_hi ( [[maybe_unused]] int i_buffer) {return 0; }
+    virtual int get_buffer_k_index_hi ( [[maybe_unused]] int i_buffer, [[maybe_unused]] const int lev) {return 0; }
     /** Set k-index of the buffer in the moving window direction currently being filled for the ith snapshot
      * \param[in] i_buffer index of the buffer
      * \param[in] kindex value used to set the k-index for the big end of the buffer box
      */
-    virtual void set_buffer_k_index_hi ( [[maybe_unused]] int i_buffer, [[maybe_unused]] int kindex) { }
+    virtual void set_buffer_k_index_hi ( [[maybe_unused]] int i_buffer, [[maybe_unused]] int kindex, [[maybe_unused]] const int lev) { }
     /** Returns lo-end of the lab-frame physical domain for the ith snapshot in BTD
      * \param[in] i_buffer index of the buffer
      * \param[in] idim dimension along which the lo-end of the lab-frame physical domain is returned

Source/Diagnostics/FlushFormats/FlushFormat.H

@@ -20,10 +20,10 @@ public:
         std::string prefix, int file_min_digits,
         bool plot_raw_fields,
         bool plot_raw_fields_guards,
-        bool use_pinned_pc = false,
+        const amrex::Vector< amrex::Geometry >& full_BTD_snapshot,
+        const bool use_pinned_pc = false,
         bool isBTD = false, int snapshotID = -1,
         int bufferID = 1, int numBuffers = 1,
-        const amrex::Geometry& full_BTD_snapshot = amrex::Geometry(),
         bool isLastBTDFlush = false,
         const amrex::Vector<int>& totalParticlesFlushedAlready = amrex::Vector<int>() ) const = 0;
 

Source/Diagnostics/FlushFormats/FlushFormatAscent.H

@@ -37,10 +37,10 @@ public:
         std::string prefix, int file_min_digits,
         bool plot_raw_fields,
         bool plot_raw_fields_guards,
-        bool use_pinned_pc = false,
+        const amrex::Vector<amrex::Geometry>& full_BTD_snapshot,
+        const bool use_pinned_pc = false,
         bool isBTD = false, int snapshotID = -1,
         int bufferID = 1, int numBuffers = 1,
-        const amrex::Geometry& full_BTD_snapshot = amrex::Geometry(),
         bool isLastBTDFlush = false,
         const amrex::Vector<int>& totalParticlesFlushedAlready = amrex::Vector<int>() ) const override;
 

Source/Diagnostics/FlushFormats/FlushFormatAscent.cpp

@@ -18,9 +18,9 @@ FlushFormatAscent::WriteToFile (
     const amrex::Vector<ParticleDiag>& particle_diags, int nlev,
     const std::string prefix, int file_min_digits, bool plot_raw_fields,
     bool plot_raw_fields_guards,
+    const amrex::Vector<amrex::Geometry>& /*full_BTD_snapshot*/,
     const bool /*use_pinned_pc*/,
     bool isBTD, int /*snapshotID*/, int /*bufferID*/, int /*numBuffers*/,
-    const amrex::Geometry& /*full_BTD_snapshot*/,
     bool /*isLastBTDFlush*/, const amrex::Vector<int>& /* totalParticlesFlushedAlready*/) const
 {
 #ifdef AMREX_USE_ASCENT

Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.H

@@ -24,10 +24,10 @@ class FlushFormatCheckpoint final : public FlushFormatPlotfile
         std::string prefix, int file_min_digits,
         bool plot_raw_fields,
         bool plot_raw_fields_guards,
-        bool use_pinned_pc = false,
+        const amrex::Vector<amrex::Geometry>& full_BTD_snapshot,
+        const bool use_pinned_pc = false,
         bool isBTD = false, int snapshotID = -1,
         int bufferID = 1, int numBuffers = 1,
-        const amrex::Geometry& full_BTD_snapshot = amrex::Geometry(),
         bool isLastBTDFlush = false,
         const amrex::Vector<int>& totalParticlesFlushedAlready = amrex::Vector<int>() ) const override final;
 

Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp

@@ -35,10 +35,10 @@ FlushFormatCheckpoint::WriteToFile (
         const std::string prefix, int file_min_digits,
         bool /*plot_raw_fields*/,
         bool /*plot_raw_fields_guards*/,
+        const amrex::Vector<amrex::Geometry>& /*full_BTD_snapshot*/,
         const bool /*use_pinned_pc*/,
         bool /*isBTD*/, int /*snapshotID*/,
         int /*bufferID*/, int /*numBuffers*/,
-        const amrex::Geometry& /*full_BTD_snapshot*/,
         bool /*isLastBTDFlush*/, const amrex::Vector<int>& /* totalParticlesFlushedAlready*/) const
 {
     WARPX_PROFILE("FlushFormatCheckpoint::WriteToFile()");

Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.H

@@ -36,10 +36,10 @@ public:
         std::string prefix, int file_min_digits,
         bool plot_raw_fields,
         bool plot_raw_fields_guards,
-        bool use_pinned_pc = false,
+        const amrex::Vector< amrex::Geometry>& full_BTD_snapshot,
+        const bool use_pinned_pc = false,
         bool isBTD = false, int snapshotID = -1,
         int bufferID = 1, int numBuffers = 1,
-        const amrex::Geometry& full_BTD_snapshot = amrex::Geometry(),
         bool isLastBTDFlush = false,
         const amrex::Vector<int>& totalParticlesFlushedAlready = amrex::Vector<int>() ) const override;
 

Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp

@@ -119,9 +119,9 @@ FlushFormatOpenPMD::WriteToFile (
     const amrex::Vector<ParticleDiag>& particle_diags, int output_levels,
     const std::string prefix, int file_min_digits, bool plot_raw_fields,
     bool plot_raw_fields_guards,
+    const amrex::Vector< amrex::Geometry>& full_BTD_snapshot,
     const bool use_pinned_pc,
     bool isBTD, int snapshotID, int bufferID, int numBuffers,
-    const amrex::Geometry& full_BTD_snapshot,
     bool isLastBTDFlush, const amrex::Vector<int>& totalParticlesFlushedAlready) const
 {
     WARPX_PROFILE("FlushFormatOpenPMD::WriteToFile()");
@@ -155,7 +155,7 @@ FlushFormatOpenPMD::WriteToFile (
 
     // fields: only dumped for coarse level
     m_OpenPMDPlotWriter->WriteOpenPMDFieldsAll(
-        varnames, mf, geom, output_levels, output_iteration, time, isBTD, full_BTD_snapshot);
+        varnames, mf, geom, output_levels, output_iteration, time, full_BTD_snapshot, isBTD);
 
     // particles: all (reside only on locally finest level)
     m_OpenPMDPlotWriter->WriteOpenPMDParticles(particle_diags, time, use_pinned_pc, isBTD, isLastBTDFlush, totalParticlesFlushedAlready);

Source/Diagnostics/FlushFormats/FlushFormatPlotfile.H

@@ -31,10 +31,10 @@ public:
         std::string prefix, int file_min_digits,
         bool plot_raw_fields,
         bool plot_raw_fields_guards,
-        bool use_pinned_pc = false,
+        const amrex::Vector< amrex::Geometry > & full_BTD_snapshot,
+        const bool use_pinned_pc = false,
         bool isBTD = false, int snapshotID = -1,
         int bufferID = 1, int numBuffers = 1,
-        const amrex::Geometry& full_BTD_snapshot = amrex::Geometry(),
         bool isLastBTDFlush = false,
         const amrex::Vector<int>& totalParticlesFlushedAlready = amrex::Vector<int>() ) const override;
 

Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp

@@ -62,9 +62,9 @@ FlushFormatPlotfile::WriteToFile (
     const amrex::Vector<ParticleDiag>& particle_diags, int nlev,
     const std::string prefix, int file_min_digits, bool plot_raw_fields,
     bool plot_raw_fields_guards,
+    const amrex::Vector<amrex::Geometry>& /*full_BTD_snapshot*/,
     const bool /*use_pinned_pc*/,
     bool isBTD, int snapshotID,  int bufferID, int numBuffers,
-    const amrex::Geometry& /*full_BTD_snapshot*/,
     bool isLastBTDFlush, const amrex::Vector<int>& /* totalParticlesFlushedAlready*/) const
 {
     WARPX_PROFILE("FlushFormatPlotfile::WriteToFile()");
@@ -320,7 +320,9 @@ FlushFormatPlotfile::WriteWarpXHeader(
                 auto& diag = warpx.GetMultiDiags().GetDiag(idiag);
                 for (int i_buffer=0; i_buffer<diag.getnumbuffers(); ++i_buffer){
                     HeaderFile << diag.gettlab(i_buffer) << "\n";
-                    HeaderFile << diag.get_buffer_k_index_hi(i_buffer) << "\n";
+                    for (int lev = 0; lev < nlevels; ++lev) {
+                        HeaderFile << diag.get_buffer_k_index_hi(i_buffer, lev) << "\n";
+                    }
                     for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
                         HeaderFile << diag.get_snapshot_domain_lo(i_buffer, idim) << "\n";
                     }

Source/Diagnostics/FlushFormats/FlushFormatSensei.H

@@ -55,10 +55,10 @@ public:
         std::string prefix, int file_min_digits,
         bool plot_raw_fields,
         bool plot_raw_fields_guards,
-        bool use_pinned_pc = false,
+        const amrex::Vector<amrex::Geometry>& full_BTD_snapshot,
+        const bool use_pinned_pc = false,
         bool isBTD = false, int snapshotID = -1,
         int bufferID = 1, int numBuffers = 1,
-        const amrex::Geometry& full_BTD_snapshot = amrex::Geometry(),
         bool isLastBTDFlush = false,
         const amrex::Vector<int>& totalParticlesFlushedAlready = amrex::Vector<int>() ) const override;
 

Source/Diagnostics/FlushFormats/FlushFormatSensei.cpp

@@ -57,9 +57,10 @@ FlushFormatSensei::WriteToFile (
     const amrex::Vector<ParticleDiag>& particle_diags,
     int nlev, const std::string prefix, int file_min_digits,
     bool plot_raw_fields, bool plot_raw_fields_guards,
+    const amrex::Vector<amrex::Geometry>& /*full_BTD_snapshot*/,
     const bool use_pinned_pc,
     bool isBTD, int /*snapshotID*/, int /*bufferID*/, int /*numBuffers*/,
-    const amrex::Geometry& /*full_BTD_snapshot*/, bool /*isLastBTDFlush*/,
+    bool /*isLastBTDFlush*/,
     const amrex::Vector<int>& totalParticlesFlushedAlready) const
 {
     amrex::ignore_unused(

Source/Diagnostics/FullDiagnostics.cpp

@@ -137,7 +137,7 @@ FullDiagnostics::Flush ( int i_buffer )
     m_flush_format->WriteToFile(
         m_varnames, m_mf_output[i_buffer], m_geom_output[i_buffer], warpx.getistep(),
         warpx.gett_new(0), m_output_species[i_buffer], nlev_output, m_file_prefix,
-        m_file_min_digits, m_plot_raw_fields, m_plot_raw_fields_guards);
+        m_file_min_digits, m_plot_raw_fields, m_plot_raw_fields_guards, m_geom_output[i_buffer]);
 
     FlushRaw();
 }

Source/Diagnostics/WarpXIO.cpp

@@ -236,8 +236,9 @@ WarpX::InitFromCheckpoint ()
                         diag.settlab(i_buffer, tlab);
                         int kindex_hi;
                         is >> kindex_hi;
-                        diag.set_buffer_k_index_hi(i_buffer, kindex_hi);
-
+                        for (int lev = 0; lev < nlevs; ++lev) {
+                            diag.set_buffer_k_index_hi(i_buffer, kindex_hi, lev);
+                        }
                         for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
                             amrex::Real snapshot_lo;
                             is >> snapshot_lo;

Source/Diagnostics/WarpXOpenPMD.H

@@ -138,10 +138,10 @@ public:
               const amrex::Vector<amrex::MultiFab>& mf,
               amrex::Vector<amrex::Geometry>& geom,
               int output_levels,
-              int iteration,
-              double time,
-              bool isBTD = false,
-              const amrex::Geometry& full_BTD_snapshot=amrex::Geometry() ) const;
+              const int iteration,
+              const double time,
+          const amrex::Vector<amrex::Geometry>& full_BTD_snapshot,
+              bool isBTD = false) const;
 
   /** Return OpenPMD File type ("bp" or "h5" or "json")*/
   std::string OpenPMDFileType () { return m_OpenPMDFileType; }

Source/Diagnostics/WarpXOpenPMD.cpp

@@ -1339,8 +1339,8 @@ WarpXOpenPMDPlot::WriteOpenPMDFieldsAll ( //const std::string& filename,
                       int output_levels,
                       const int iteration,
                       const double time,
-                      bool isBTD,
-                      const amrex::Geometry& full_BTD_snapshot ) const
+              const amrex::Vector<amrex::Geometry>& full_BTD_snapshot,
+                      bool isBTD ) const
 {
     //This is AMReX's tiny profiler. Possibly will apply it later
     WARPX_PROFILE("WarpXOpenPMDPlot::WriteOpenPMDFields()");
@@ -1371,7 +1371,7 @@ WarpXOpenPMDPlot::WriteOpenPMDFieldsAll ( //const std::string& filename,
     for (int lev=0; lev < output_levels; lev++) {
         amrex::Geometry full_geom = geom[lev];
         if( isBTD )
-            full_geom = full_BTD_snapshot;
+            full_geom = full_BTD_snapshot[lev];
 
         // setup is called once. So it uses property "period" from first
         // geometry for <all> field levels.