WIP: Prototype for "poor man's" load balancing

include/picongpu/plugins/Checkpoint.x.cpp

@@ -123,9 +123,37 @@ namespace picongpu
         void checkpoint(uint32_t currentStep, std::string const checkpointDirectory) override
         {
             auto cBackend = ioBackends.find(checkpointBackendName);
-            if(cBackend != ioBackends.end())
+            if(cBackend == ioBackends.end())
             {
-                cBackend->second->dumpCheckpoint(currentStep, checkpointDirectory, checkpointFilename);
+                return;
+            }
+
+            int mpiInitialized;
+            MPI_CHECK(MPI_Query_thread(&mpiInitialized));
+
+            if(mpiInitialized < MPI_THREAD_MULTIPLE)
+            {
+                cBackend->second->dumpCheckpoint(currentStep, checkpointDirectory, checkpointFilename, std::nullopt);
+            }
+            else
+            {
+                std::atomic<signed int> synchronization = 0;
+
+                // need to copy the plugin temporarily to avoid race conditions
+                auto restartFuture = std::async(
+                    std::launch::async,
+                    [this, copiedBackend = cBackend, currentStep, &checkpointDirectory, &synchronization]()
+                    {
+                        copiedBackend->second->doRestart(
+                            currentStep,
+                            checkpointDirectory,
+                            this->checkpointFilename,
+                            this->restartChunkSize,
+                            {&synchronization});
+                    });
+                cBackend->second
+                    ->dumpCheckpoint(currentStep, checkpointDirectory, checkpointFilename, {&synchronization});
+                restartFuture.wait();
             }
         }
 
@@ -134,7 +162,8 @@ namespace picongpu
             auto rBackend = ioBackends.find(restartBackendName);
             if(rBackend != ioBackends.end())
             {
-                rBackend->second->doRestart(restartStep, restartDirectory, restartFilename, restartChunkSize);
+                rBackend->second
+                    ->doRestart(restartStep, restartDirectory, restartFilename, restartChunkSize, std::nullopt);
             }
         }
 

include/picongpu/plugins/openPMD/NDScalars.hpp

@@ -89,7 +89,7 @@ namespace picongpu
                     localDomainOffset[d] = Environment<simDim>::get().GridController().getPosition()[d];
                 }
 
-                ::openPMD::Series& series = *params.openPMDSeries;
+                ::openPMD::Series& series = *params.writeOpenPMDSeries;
                 ::openPMD::MeshRecordComponent mrc
                     = series.writeIterations()[currentStep].meshes[baseName + "_" + group][dataset];
 
@@ -125,7 +125,7 @@ namespace picongpu
                     std::make_shared<T_Scalar>(value),
                     std::move(std::get<1>(tuple)),
                     std::move(std::get<2>(tuple)));
-                params.openPMDSeries->flush(PreferredFlushTarget::Buffer);
+                std::get<0>(tuple).seriesFlush(PreferredFlushTarget::Buffer);
             }
 
         private:
@@ -162,7 +162,7 @@ namespace picongpu
 
 
                 auto datasetName = baseName + "/" + group + "/" + dataset;
-                ::openPMD::Series& series = *params.openPMDSeries;
+                ::openPMD::Series& series = *params.readOpenPMDSeries;
                 ::openPMD::MeshRecordComponent mrc
                     = series.iterations[currentStep].open().meshes[baseName + "_" + group][dataset];
                 auto ndim = mrc.getDimensionality();
@@ -174,11 +174,12 @@ namespace picongpu
                 DataSpace<simDim> gridPos = Environment<simDim>::get().GridController().getPosition();
                 ::openPMD::Offset start;
                 ::openPMD::Extent count;
+                ::openPMD::Extent extent = mrc.getExtent();
                 start.reserve(ndim);
                 count.reserve(ndim);
                 for(int d = 0; d < ndim; ++d)
                 {
-                    start.push_back(gridPos.revert()[d]);
+                    start.push_back(gridPos.revert()[d] % extent[d]); // well well well, TODO fix this but how
                     count.push_back(1);
                 }
 

include/picongpu/plugins/openPMD/openPMDWriter.def

@@ -79,7 +79,9 @@ namespace picongpu
         struct ThreadParams : PluginParameters
         {
 #if (ENABLE_OPENPMD == 1)
-            std::unique_ptr<::openPMD::Series> openPMDSeries; /* is null iff there is no series currently open */
+            // might be needed at the same time for load balancing purposes
+            std::optional<::openPMD::Series> writeOpenPMDSeries; /* is null iff there is no series currently open */
+            std::optional<::openPMD::Series> readOpenPMDSeries; /* is null iff there is no series currently open */
 #endif
             /** current dump is a checkpoint */
             bool isCheckpoint;
@@ -161,7 +163,7 @@ namespace picongpu
 #endif
             };
 
-            void closeSeries();
+            void closeSeries(::openPMD::Access);
 
             /*
              * If file is empty, read from command line parameters.