Move sensei to new diagnostics (#1040)

MaxThevenet · burlen · web-flow · commit bd7d6f6dd646 · 2020-05-29T17:42:10.000-07:00
* move ascent to new diagnostics

* move Sensei to the new diagnostics

* minor cleaning in new diagnostics

* oops, had forgotten some files

* fix typo

* another typo introduced when merging master into this branch

* FlushFormat::WriteToFile takes Vector&lt;MultiFab&gt;&amp; instead of Vector&lt;const MultiFab*&gt;

* IO output multifab has 1 guard cell when using sensei

* updates to FlushFormatSensei

* add some documentation to the class
* add refrence to AmrMesh instance needed by the adaptor
* add some error output in the case WarpX was not compiled w/ SENSEI
* tested with SENSEI 3.2.0

Co-authored-by: Burlen Loring &lt;bloring@lbl.gov&gt;
diff --git a/Source/Diagnostics/Diagnostics.cpp b/Source/Diagnostics/Diagnostics.cpp
@@ -7,6 +7,7 @@
 #include "FlushFormats/FlushFormatPlotfile.H"
 #include "FlushFormats/FlushFormatCheckpoint.H"
 #include "FlushFormats/FlushFormatAscent.H"
+#include "FlushFormats/FlushFormatSensei.H"
 #ifdef WARPX_USE_OPENPMD
 #   include "FlushFormats/FlushFormatOpenPMD.H"
 #endif
@@ -39,8 +40,10 @@ Diagnostics::ReadParameters ()
     m_intervals = IntervalsParser(period_string);
     pp.query("format", m_format);
     AMREX_ALWAYS_ASSERT_WITH_MESSAGE(
-        m_format == "plotfile" || m_format == "openpmd" || m_format == "checkpoint" || m_format == "ascent",
-        "<diag>.format must be plotfile or openpmd or checkpoint or ascent");
+        m_format == "plotfile" || m_format == "openpmd" ||
+        m_format == "checkpoint" || m_format == "ascent" ||
+        m_format == "sensei",
+        "<diag>.format must be plotfile or openpmd or checkpoint or ascent or sensei");
     bool raw_specified = pp.query("plot_raw_fields", m_plot_raw_fields);
     raw_specified += pp.query("plot_raw_fields_guards", m_plot_raw_fields_guards);
     bool varnames_specified = pp.queryarr("fields_to_plot", m_varnames);
@@ -148,6 +151,14 @@ Diagnostics::InitData ()
         m_flush_format = new FlushFormatCheckpoint;
     } else if (m_format == "ascent"){
         m_flush_format = new FlushFormatAscent;
+    } else if (m_format == "sensei"){
+#ifdef BL_USE_SENSEI_INSITU
+        m_flush_format = new FlushFormatSensei(
+            dynamic_cast<amrex::AmrMesh*>(const_cast<WarpX*>(&warpx)),
+            m_diag_name);
+#else
+        amrex::Abort("To use SENSEI in situ, compile with USE_SENSEI=TRUE");
+#endif
     } else if (m_format == "openpmd"){
 #ifdef WARPX_USE_OPENPMD
         m_flush_format = new FlushFormatOpenPMD(m_diag_name);
@@ -195,7 +206,7 @@ Diagnostics::Flush ()
 {
     auto & warpx = WarpX::GetInstance();
     m_flush_format->WriteToFile(
-        m_varnames, GetVecOfConstPtrs(m_mf_output), warpx.Geom(), warpx.getistep(),
+        m_varnames, m_mf_output, warpx.Geom(), warpx.getistep(),
         warpx.gett_new(0), m_all_species, nlev, m_file_prefix,
         m_plot_raw_fields, m_plot_raw_fields_guards, m_plot_raw_rho, m_plot_raw_F);
 }
@@ -401,7 +412,8 @@ Diagnostics::DefineDiagMultiFab ( int lev ) {
     // is different from the lo and hi physical co-ordinates of the simulation domain.
     if (use_warpxba == false) dmap = DistributionMapping{ba};
     // Allocate output MultiFab for diagnostics. The data will be stored at cell-centers.
-    m_mf_output[lev] = MultiFab(ba, dmap, m_varnames.size(), 0);
+    int ngrow = (m_format == "sensei") ? 1 : 0;
+    m_mf_output[lev] = MultiFab(ba, dmap, m_varnames.size(), ngrow);
 }
 
 
diff --git a/Source/Diagnostics/FlushFormats/FlushFormat.H b/Source/Diagnostics/FlushFormats/FlushFormat.H
@@ -10,7 +10,7 @@ public:
     /** Flush fields and particles to file */
     virtual void WriteToFile (
         const amrex::Vector<std::string> varnames,
-        const amrex::Vector<const amrex::MultiFab*> mf,
+        const amrex::Vector<amrex::MultiFab>& mf,
         amrex::Vector<amrex::Geometry>& geom,
         const amrex::Vector<int> iteration, const double time,
         const amrex::Vector<ParticleDiag>& particle_diags, int nlev,
diff --git a/Source/Diagnostics/FlushFormats/FlushFormatAscent.H b/Source/Diagnostics/FlushFormats/FlushFormatAscent.H
@@ -4,7 +4,6 @@
 #include "FlushFormat.H"
 #include "Diagnostics/ParticleDiag/ParticleDiag.H"
 
-
 #ifdef AMREX_USE_ASCENT
 #   include <ascent.hpp>
 #   include <AMReX_Conduit_Blueprint.H>
@@ -21,7 +20,7 @@ public:
     /** Do in-situ visualization for field and particle data */
     virtual void WriteToFile (
         const amrex::Vector<std::string> varnames,
-        const amrex::Vector<const amrex::MultiFab*> mf,
+        const amrex::Vector<amrex::MultiFab>& mf,
         amrex::Vector<amrex::Geometry>& geom,
         const amrex::Vector<int> iteration, const double time,
         const amrex::Vector<ParticleDiag>& particle_diags, int nlev, const std::string prefix,
diff --git a/Source/Diagnostics/FlushFormats/FlushFormatAscent.cpp b/Source/Diagnostics/FlushFormats/FlushFormatAscent.cpp
@@ -6,7 +6,7 @@ using namespace amrex;
 void
 FlushFormatAscent::WriteToFile (
     const amrex::Vector<std::string> varnames,
-    const amrex::Vector<const amrex::MultiFab*> mf,
+    const amrex::Vector<amrex::MultiFab>& mf,
     amrex::Vector<amrex::Geometry>& geom,
     const amrex::Vector<int> iteration, const double time,
     const amrex::Vector<ParticleDiag>& particle_diags, int nlev,
@@ -20,7 +20,7 @@ FlushFormatAscent::WriteToFile (
     // wrap mesh data
     conduit::Node bp_mesh;
     amrex::MultiLevelToBlueprint(
-        nlev, mf, varnames, geom, time, iteration, warpx.refRatio(), bp_mesh);
+        nlev, amrex::GetVecOfConstPtrs(mf), varnames, geom, time, iteration, warpx.refRatio(), bp_mesh);
 
     WriteParticles(particle_diags, bp_mesh);
 
diff --git a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.H b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.H
@@ -8,7 +8,7 @@ class FlushFormatCheckpoint final : public FlushFormatPlotfile
     /** Flush fields and particles to plotfile */
     virtual void WriteToFile (
         const amrex::Vector<std::string> varnames,
-        const amrex::Vector<const amrex::MultiFab*> mf,
+        const amrex::Vector<amrex::MultiFab>& mf,
         amrex::Vector<amrex::Geometry>& geom,
         const amrex::Vector<int> iteration, const double time,
         const amrex::Vector<ParticleDiag>& particle_diags, int nlev, const std::string prefix,
diff --git a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp
@@ -14,7 +14,7 @@ namespace
 void
 FlushFormatCheckpoint::WriteToFile (
         const amrex::Vector<std::string> varnames,
-        const amrex::Vector<const amrex::MultiFab*> mf,
+        const amrex::Vector<amrex::MultiFab>& mf,
         amrex::Vector<amrex::Geometry>& geom,
         const amrex::Vector<int> iteration, const double time,
         const amrex::Vector<ParticleDiag>& particle_diags, int nlev, const std::string prefix,
diff --git a/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.H b/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.H
@@ -19,7 +19,7 @@ public:
     /** Flush fields and particles to plotfile */
     virtual void WriteToFile (
         const amrex::Vector<std::string> varnames,
-        const amrex::Vector<const amrex::MultiFab*> mf,
+        const amrex::Vector<amrex::MultiFab>& mf,
         amrex::Vector<amrex::Geometry>& geom,
         const amrex::Vector<int> iteration, const double time,
         const amrex::Vector<ParticleDiag>& particle_diags, int nlev, const std::string prefix,
diff --git a/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp b/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp
@@ -6,11 +6,6 @@
 
 using namespace amrex;
 
-namespace
-{
-    const std::string level_prefix {"Level_"};
-}
-
 FlushFormatOpenPMD::FlushFormatOpenPMD (const std::string& diag_name)
 {
     ParmParse pp(diag_name);
@@ -29,7 +24,7 @@ FlushFormatOpenPMD::FlushFormatOpenPMD (const std::string& diag_name)
 void
 FlushFormatOpenPMD::WriteToFile (
     const amrex::Vector<std::string> varnames,
-    const amrex::Vector<const amrex::MultiFab*> mf,
+    const amrex::Vector<amrex::MultiFab>& mf,
     amrex::Vector<amrex::Geometry>& geom,
     const amrex::Vector<int> iteration, const double time,
     const amrex::Vector<ParticleDiag>& particle_diags, int nlev,
@@ -47,7 +42,7 @@ FlushFormatOpenPMD::WriteToFile (
 
     // fields: only dumped for coarse level
     m_OpenPMDPlotWriter->WriteOpenPMDFields(
-        varnames, *mf[0], geom[0], iteration[0], time);
+        varnames, mf[0], geom[0], iteration[0], time);
 
     // particles: all (reside only on locally finest level)
     m_OpenPMDPlotWriter->WriteOpenPMDParticles(particle_diags);
diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.H b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.H
@@ -15,7 +15,7 @@ public:
     /** Flush fields and particles to plotfile */
     virtual void WriteToFile (
         const amrex::Vector<std::string> varnames,
-        const amrex::Vector<const amrex::MultiFab*> mf,
+        const amrex::Vector<amrex::MultiFab>& mf,
         amrex::Vector<amrex::Geometry>& geom,
         const amrex::Vector<int> iteration, const double time,
         const amrex::Vector<ParticleDiag>& particle_diags, int nlev, const std::string prefix,
diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp
@@ -15,7 +15,7 @@ namespace
 void
 FlushFormatPlotfile::WriteToFile (
     const amrex::Vector<std::string> varnames,
-    const amrex::Vector<const amrex::MultiFab*> mf,
+    const amrex::Vector<amrex::MultiFab>& mf,
     amrex::Vector<amrex::Geometry>& geom,
     const amrex::Vector<int> iteration, const double time,
     const amrex::Vector<ParticleDiag>& particle_diags, int nlev,
@@ -31,7 +31,7 @@ FlushFormatPlotfile::WriteToFile (
     VisMF::SetHeaderVersion(amrex::VisMF::Header::Version_v1);
     if (plot_raw_fields) rfs.emplace_back("raw_fields");
     amrex::WriteMultiLevelPlotfile(filename, nlev,
-                                   mf,
+                                   amrex::GetVecOfConstPtrs(mf),
                                    varnames, geom,
                                    time, iteration, warpx.refRatio(),
                                    "HyperCLaw-V1.1",
diff --git a/Source/Diagnostics/FlushFormats/FlushFormatSensei.H b/Source/Diagnostics/FlushFormats/FlushFormatSensei.H
@@ -0,0 +1,70 @@
+#ifndef WARPX_FLUSHFORMATSENSEI_H_
+#define WARPX_FLUSHFORMATSENSEI_H_
+
+#include "FlushFormat.H"
+#include "AMReX_AmrMesh.H"
+
+#if defined(BL_USE_SENSEI_INSITU)
+#include "AMReX_AmrMeshInSituBridge.H"
+#else
+namespace amrex {
+using AmrMeshInSituBridge = void;
+}
+#endif
+
+/**
+ * \brief This class aims at dumping performing in-situ diagnostics with
+ * SENSEI.  In particular, function WriteToFile takes fields and particles as
+ * input arguments, and calls amrex functions to do the in-situ visualization.
+ *
+ * To use the SENSEI flush format one must compile WarpX with the GNUMakefile
+ * setting:  USE_SENSEI_INSITU=TRUE
+ *
+ * In addition to the standard controls, the following inputs file fields are
+ * supported:
+ *
+ *      sensei_config - the path to a SENSEI XML configuration (required)
+ *      sensei_pin_mesh - integer 0 or 1 forcing the moving mesh to be fixed
+ *                        at 0,0,0 (optional)
+ *
+ * Fri 29 May 2020 11:19:38 AM PDT : Tested with SENSEI version 3.2.0
+ */
+class FlushFormatSensei : public FlushFormat
+{
+public:
+    FlushFormatSensei();
+    ~FlushFormatSensei();
+
+    FlushFormatSensei(const FlushFormatSensei &) = delete;
+    void operator=(const FlushFormatSensei &) = delete;
+
+    /** \breif Construct.
+     * \parm[in] amr_mesh an AmrMesh instance
+     * \parm[in] diag_name ParmParse scope string.
+     */
+    FlushFormatSensei (amrex::AmrMesh *amr_mesh, std::string diag_name);
+
+    /** Do in-situ visualization for field and particle data */
+    virtual void WriteToFile (
+        const amrex::Vector<std::string> varnames,
+        const amrex::Vector<amrex::MultiFab>& mf,
+        amrex::Vector<amrex::Geometry>& geom,
+        const amrex::Vector<int> iteration, const double time,
+        const amrex::Vector<ParticleDiag>& particle_diags, int nlev, const std::string prefix,
+        bool plot_raw_fields,
+        bool plot_raw_fields_guards,
+        bool plot_raw_rho, bool plot_raw_F) const override;
+
+    /** \brief Do in-situ visualization for particle data.
+     * \param[in] particle_diags Each element of this vector handles output of 1 species.
+     */
+    void WriteParticles(const amrex::Vector<ParticleDiag>& particle_diags) const;
+
+private:
+    amrex::AmrMesh * m_amr_mesh;
+    amrex::AmrMeshInSituBridge * m_insitu_bridge;
+    std::string m_insitu_config;
+    int m_insitu_pin_mesh;
+};
+
+#endif // WARPX_FLUSHFORMATSENSEI_H_
diff --git a/Source/Diagnostics/FlushFormats/FlushFormatSensei.cpp b/Source/Diagnostics/FlushFormats/FlushFormatSensei.cpp
@@ -0,0 +1,99 @@
+#include "FlushFormatSensei.H"
+#include "WarpX.H"
+
+#ifdef BL_USE_SENSEI_INSITU
+# include <AMReX_AmrMeshInSituBridge.H>
+#endif
+
+FlushFormatSensei::FlushFormatSensei () :
+    m_insitu_config(), m_insitu_pin_mesh(0), m_insitu_bridge(nullptr),
+    m_amr_mesh(nullptr)
+{}
+
+FlushFormatSensei::FlushFormatSensei (amrex::AmrMesh *amr_mesh,
+    std::string diag_name) :
+    m_insitu_config(), m_insitu_pin_mesh(0), m_insitu_bridge(nullptr),
+    m_amr_mesh(amr_mesh)
+{
+#ifndef BL_USE_SENSEI_INSITU
+    (void)amr_mesh;
+    (void)diag_name;
+#else
+    amrex::ParmParse pp(diag_name);
+
+    pp.query("sensei_config", m_insitu_config);
+    pp.query("sensei_pin_mesh", m_insitu_pin_mesh);
+
+    m_insitu_bridge = new amrex::AmrMeshInSituBridge;
+    m_insitu_bridge->setEnabled(true);
+    m_insitu_bridge->setConfig(m_insitu_config);
+    m_insitu_bridge->setPinMesh(m_insitu_pin_mesh);
+    if (!m_amr_mesh || m_insitu_bridge->initialize())
+    {
+        amrex::ErrorStream() << "FlushFormtSensei::FlushFormatSensei : "
+            "Failed to initialize the in situ bridge." << std::endl;
+
+        amrex::Abort();
+    }
+    m_insitu_bridge->setFrequency(1);
+#endif
+}
+
+FlushFormatSensei::~FlushFormatSensei ()
+{
+#ifdef BL_USE_SENSEI_INSITU
+    delete m_insitu_bridge;
+#endif
+}
+
+void
+FlushFormatSensei::WriteToFile (
+    const amrex::Vector<std::string> varnames,
+    const amrex::Vector<amrex::MultiFab>& mf,
+    amrex::Vector<amrex::Geometry>& geom,
+    const amrex::Vector<int> iteration, const double time,
+    const amrex::Vector<ParticleDiag>& particle_diags, int nlev,
+    const std::string prefix, bool plot_raw_fields,
+    bool plot_raw_fields_guards, bool plot_raw_rho, bool plot_raw_F) const
+{
+#ifndef BL_USE_SENSEI_INSITU
+    (void)varnames;
+    (void)mf;
+    (void)geom;
+    (void)iteration;
+    (void)time;
+    (void)particle_diags;
+    (void)nlev;
+    (void)prefix;
+    (void)plot_raw_fields;
+    (void)plot_raw_fields_guards;
+    (void)plot_raw_rho;
+    (void)plot_raw_F;
+#else
+    amrex::Vector<amrex::MultiFab> *mf_ptr =
+        const_cast<amrex::Vector<amrex::MultiFab>*>(&mf);
+
+    if (m_insitu_bridge->update(iteration[0], time, m_amr_mesh,
+        {mf_ptr}, {varnames}))
+    {
+        amrex::ErrorStream() << "FlushFormatSensei::WriteToFile : "
+            "Failed to update the in situ bridge." << std::endl;
+
+        amrex::Abort();
+    }
+#endif
+}
+
+void
+FlushFormatSensei::WriteParticles (
+    const amrex::Vector<ParticleDiag>& particle_diags) const
+{
+#ifndef BL_USE_SENSEI_INSITU
+    (void)particle_diags;
+#else
+    amrex::ErrorStream() << "FlushFormatSensei::WriteParticles : "
+        "Not yet implemented." << std::endl;
+
+    amrex::Abort();
+#endif
+}
diff --git a/Source/Diagnostics/FlushFormats/Make.package b/Source/Diagnostics/FlushFormats/Make.package
@@ -1,6 +1,7 @@
 CEXE_sources += FlushFormatPlotfile.cpp
 CEXE_sources += FlushFormatCheckpoint.cpp
 CEXE_sources += FlushFormatAscent.cpp
+CEXE_sources += FlushFormatSensei.cpp
 ifeq ($(USE_OPENPMD), TRUE)
     CEXE_sources += FlushFormatOpenPMD.cpp
 endif
diff --git a/Source/Diagnostics/WarpXIO.cpp b/Source/Diagnostics/WarpXIO.cpp
@@ -278,32 +278,3 @@ WarpX::GetCellCenteredData() {
 
     return std::move(cc[0]);
 }
-
-void
-WarpX::UpdateInSitu () const
-{
-#if defined(BL_USE_SENSEI_INSITU)
-    WARPX_PROFILE("WarpX::UpdateInSitu()");
-
-    // Average the fields from the simulation to the cell centers
-    const int ngrow = 1;
-    Vector<std::string> varnames; // Name of the written fields
-    // mf_avg will contain the averaged, cell-centered fields
-    Vector<MultiFab> mf_avg;
-    WarpX::AverageAndPackFields( varnames, mf_avg, ngrow );
-
-#   ifdef BL_USE_SENSEI_INSITU
-    if (insitu_bridge->update(istep[0], t_new[0],
-        dynamic_cast<amrex::AmrMesh*>(const_cast<WarpX*>(this)),
-        {&mf_avg}, {varnames}))
-    {
-        amrex::ErrorStream()
-            << "WarpXIO::UpdateInSitu : Failed to update the in situ bridge."
-            << std::endl;
-
-        amrex::Abort();
-    }
-#   endif
-
-#endif
-}
diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp
diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp
diff --git a/Source/WarpX.H b/Source/WarpX.H
diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp
