lustre workaround

Author: BenWibking

Src/Base/AMReX_PlotFileUtil.cpp

@@ -1,4 +1,5 @@
 
+#include "AMReX_ParallelDescriptor.H"
 #include <AMReX_VisMF.H>
 #include <AMReX_AsyncOut.H>
 #include <AMReX_PlotFileUtil.H>
@@ -236,6 +237,7 @@ WriteMultiLevelPlotfile (const std::string& plotfilename, int nlevels,
             }
             VisMF::Write(*data, MultiFabFileFullPrefix(level, plotfilename, levelPrefix, mfPrefix));
         }
+        ParallelDescriptor::Barrier();
     }
 }
 

Src/Base/AMReX_VisMF.cpp

@@ -1111,7 +1111,7 @@ VisMF::Write (const FabArray<FArrayBox>&    mf,
             nfi.Stream().flush();
             delete [] allFabData;
 
-        } else {    // ---- write fabs individually
+          } else {    // ---- write fabs individually
             for(MFIter mfi(mf); mfi.isValid(); ++mfi) {
                 std::streamoff hLength = 0;
                 const FArrayBox &fab = mf[mfi];
@@ -1123,10 +1123,9 @@ VisMF::Write (const FabArray<FArrayBox>&    mf,
                     hLength = static_cast<std::streamoff>(hss.tellp());
                     auto tstr = hss.str();
                     nfi.Stream().write(tstr.c_str(), hLength);    // ---- the fab header
-                    nfi.Stream().flush();
                 }
                 Real const* fabdata = fab.dataPtr();
-#ifdef AMREX_USE_GPU
+        #ifdef AMREX_USE_GPU
                 std::unique_ptr<FArrayBox> hostfab;
                 if (fab.arena()->isManaged() || fab.arena()->isDevice()) {
                     hostfab = std::make_unique<FArrayBox>(fab.box(), fab.nComp(),
@@ -1136,20 +1135,19 @@ VisMF::Write (const FabArray<FArrayBox>&    mf,
                     Gpu::streamSynchronize();
                     fabdata = hostfab->dataPtr();
                 }
-#endif
+        #endif
                 if(doConvert) {
                     char *cDataPtr = new char[writeDataSize];
                     RealDescriptor::convertFromNativeFormat(static_cast<void *> (cDataPtr),
                                                             writeDataItems,
                                                             fabdata, *whichRD);
                     nfi.Stream().write(cDataPtr, writeDataSize);
-                    nfi.Stream().flush();
                     delete [] cDataPtr;
                 } else {    // ---- copy from the fab
                     nfi.Stream().write((char *) fabdata, writeDataSize);
-                    nfi.Stream().flush();
                 }
             }
+            nfi.Stream().flush();
         }
     }
 

Src/Particle/AMReX_ParticleIO.H

@@ -613,10 +613,10 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssig
                                     particle_io_flags, tile_map[grid], count[grid], is_checkpoint);
 
         writeIntData(istuff.dataPtr(), istuff.size(), ofs);
-        ofs.flush();  // Some systems require this flush() (probably due to a bug)
+        //ofs.flush();  // Some systems require this flush() (probably due to a bug)
 
         WriteParticleRealData(rstuff.dataPtr(), rstuff.size(), ofs);
-        ofs.flush();  // Some systems require this flush() (probably due to a bug)
+        //ofs.flush();  // Some systems require this flush() (probably due to a bug)
     }
 }
 

Src/Particle/AMReX_WriteBinaryParticleData.H

@@ -669,6 +669,7 @@ void WriteBinaryParticleDataSync (PC const& pc,
                 }
             }
         }
+        ParallelDescriptor::Barrier();
     }
 
     if (ParallelDescriptor::IOProcessor())