chunked Packing for diagnostics

src/amr/data/particles/particles_data.hpp

@@ -5,6 +5,7 @@
 #include <cstddef>
 #include <numeric>
 #include <stdexcept>
+#include <tuple>
 #include <vector>
 
 #include "core/def/phare_mpi.hpp"
@@ -159,14 +160,13 @@ namespace amr
 
                 particles.sortMapping();
 
-                Packer packer(particles);
-                core::ContiguousParticles<dim> soa{particles.size()};
-                packer.pack(soa);
-
-                std::size_t part_idx = 0;
-                core::apply(soa.as_tuple(), [&](auto const& arg) {
-                    restart_db->putVector(name + "_" + packer.keys()[part_idx++], arg);
-                });
+                Packer{particles}.pack_ranges_into(
+                    [&](auto const& arr, auto const...) {
+                        core::double_apply(arr(), [&](auto const& key, auto const& arg) {
+                            restart_db->putVector(name + "_" + key, arg);
+                        });
+                    },
+                    particles.size()); // no means from samrai to write in chunks
             };
 
             putParticles("domainParticles", domainParticles);
@@ -201,12 +201,9 @@ namespace amr
                     = restart_db->getArraySize(name + "_" + Packer::arbitrarySingleValueKey());
                 core::ContiguousParticles<dim> soa{n_particles};
 
-                {
-                    std::size_t part_idx = 0;
-                    core::apply(soa.as_tuple(), [&](auto& arg) {
-                        restart_db->getVector(name + "_" + Packer::keys()[part_idx++], arg);
-                    });
-                }
+                core::double_apply(soa(), [&](auto const& key, auto& arg) {
+                    restart_db->getVector(name + "_" + key, arg);
+                });
 
                 assert(particles.size() == 0);
                 particles.reserve(n_particles);

src/core/data/particles/particle_array.hpp

@@ -3,6 +3,8 @@
 
 
 #include <cstddef>
+#include <cstdint>
+#include <tuple>
 #include <utility>
 #include <vector>
 
@@ -266,7 +268,7 @@ namespace core
         using container_t = std::conditional_t<OwnedState, std::vector<T>, Span<T>>;
 
         template<bool OS = OwnedState, typename = std::enable_if_t<OS>>
-        ContiguousParticles(std::size_t s)
+        ContiguousParticles(std::size_t s = 0)
             : iCell(s * dim)
             , delta(s * dim)
             , weight(s)
@@ -338,13 +340,48 @@ namespace core
             std::vector<ParticleView<dim>> views;
         };
 
-        NO_DISCARD auto as_tuple()
+
+        template<typename T>
+        using tuple_element_t = std::tuple<std::string, container_t<T>&>;
+
+        auto operator()()
+        {
+            return std::make_tuple(
+                tuple_element_t<double>("weight", weight),
+                tuple_element_t<double>("charge", charge), tuple_element_t<int>("iCell", iCell),
+                tuple_element_t<double>("delta", delta), tuple_element_t<double>("v", v));
+        }
+
+        template<typename T>
+        using tuple_element_ct = std::tuple<std::string, container_t<T> const&>;
+
+        auto operator()() const
         {
-            return std::forward_as_tuple(weight, charge, iCell, delta, v);
+            return std::make_tuple(
+                tuple_element_ct<double>("weight", weight),
+                tuple_element_ct<double>("charge", charge), tuple_element_ct<int>("iCell", iCell),
+                tuple_element_ct<double>("delta", delta), tuple_element_ct<double>("v", v));
         }
-        NO_DISCARD auto as_tuple() const
+
+
+        void clear()
+        {
+            if constexpr (OwnedState)
+                core::double_apply((*this)(), [](auto& key, auto& vec) { vec.clear(); });
+        }
+
+        void push_back(auto const& particle)
         {
-            return std::forward_as_tuple(weight, charge, iCell, delta, v);
+            weight.push_back(particle.weight);
+            charge.push_back(particle.charge);
+
+            for (std::uint16_t di = 0; di < dim; ++di)
+            {
+                iCell.push_back(particle.iCell[di]);
+                delta.push_back(particle.delta[di]);
+            }
+            for (std::uint16_t di = 0; di < 3; ++di)
+                v.push_back(particle.v[di]);
         }
 
         NO_DISCARD auto begin() { return iterator(this); }

src/core/data/particles/particle_packer.hpp

@@ -2,12 +2,11 @@
 #define PHARE_CORE_DATA_PARTICLE_PACKER_HPP
 
 
-#include <cstddef>
-#include <vector>
-
+#include "core/def.hpp"
 #include "particle.hpp"
 #include "particle_array.hpp"
-#include "core/def.hpp"
+
+#include <cstddef>
 
 namespace PHARE::core
 {
@@ -24,7 +23,7 @@ class ParticlePacker
     {
     }
 
-    NO_DISCARD static auto get(Particle<dim> const& particle)
+    NO_DISCARD static constexpr auto get(Particle<dim> const& particle)
     {
         return std::forward_as_tuple(particle.weight, particle.charge, particle.iCell,
                                      particle.delta, particle.v);
@@ -42,21 +41,34 @@ class ParticlePacker
     NO_DISCARD bool hasNext() const { return it_ < particles_.size(); }
     NO_DISCARD auto next() { return get(it_++); }
 
-    void pack(ContiguousParticles<dim>& copy)
+    void pack(ContiguousParticles<dim>& soa) const
     {
-        auto copyTo = [](auto& a, auto& idx, auto size, auto& v) {
-            std::copy(a.begin(), a.begin() + size, v.begin() + (idx * size));
-        };
-        std::size_t idx = 0;
-        while (this->hasNext())
+        for (auto const& particle : particles_)
+            soa.push_back(particle);
+    }
+
+    template<typename Fn>
+    void pack_ranges_into(Fn const fn, std::size_t const S = 2048) const
+    {
+        ContiguousParticles<dim> soa{S};
+        soa.clear(); // reserved but 0 size
+
+        std::size_t i = 0;
+        for (; i < particles_.size() / S; ++i)
+        {
+            std::size_t const pi = i * S;
+            for (std::size_t bi = 0; bi < S; ++bi)
+                soa.push_back(particles_[pi + bi]);
+            fn(soa, pi);
+            soa.clear();
+        }
+
+        if (auto const remaining = particles_.size() % S)
         {
-            auto next        = this->next();
-            copy.weight[idx] = std::get<0>(next);
-            copy.charge[idx] = std::get<1>(next);
-            copyTo(std::get<2>(next), idx, dim, copy.iCell);
-            copyTo(std::get<3>(next), idx, dim, copy.delta);
-            copyTo(std::get<4>(next), idx, 3, copy.v);
-            idx++;
+            std::size_t const pi = i * S;
+            for (std::size_t bi = 0; bi < remaining; ++bi)
+                soa.push_back(particles_[pi + bi]);
+            fn(soa, pi);
         }
     }
 

src/core/utilities/types.hpp

@@ -102,6 +102,16 @@ namespace core
         std::apply([&](auto&... args) { (func(args), ...); }, tuple);
     }
 
+
+    template<typename TupleOfTuples, typename Func>
+    void double_apply(TupleOfTuples&& tuples, Func&& func)
+    {
+        auto on_tuple
+            = [&](auto& tuple) { std::apply([&](auto&&... args) { func(args...); }, tuple); };
+
+        std::apply([&](auto&&... args) { (on_tuple(args), ...); }, tuples);
+    }
+
     template<typename Type, std::size_t Size> // std::array::fill is only constexpr in C++20 ffs
     constexpr void fill(Type value, std::array<Type, Size>& array)
     {
@@ -226,7 +236,7 @@ namespace core
 
     NO_DISCARD inline std::optional<std::string> get_env(std::string const& key)
     {
-        if (const char* val = std::getenv(key.c_str()))
+        if (char const* val = std::getenv(key.c_str()))
             return std::string{val};
         return std::nullopt;
     }

src/diagnostic/detail/h5writer.hpp

@@ -162,7 +162,7 @@ class H5Writer
     static void writeTensorFieldAsDataset(HighFiveFile& h5, std::string path, TensorField& tField)
     {
         for (auto& [id, type] : core::Components::componentMap<TensorField::rank>())
-            h5.write_data_set_flat<dimension>(path + "_" + id, tField.getComponent(type).data());
+            h5.write_data_set_flat(path + "_" + id, tField.getComponent(type).data());
     }
 
     auto& modelView() { return modelView_; }

src/diagnostic/detail/types/fluid.hpp

@@ -296,9 +296,7 @@ void FluidDiagnosticWriter<H5Writer>::write(DiagnosticProperties& diagnostic)
     auto& ions     = h5Writer.modelView().getIons();
     auto& h5file   = *fileData_.at(diagnostic.quantity);
 
-    auto writeDS = [&](auto path, auto& field) {
-        h5file.template write_data_set_flat<GridLayout::dimension>(path, field.data());
-    };
+    auto writeDS = [&](auto path, auto& field) { h5file.write_data_set_flat(path, field.data()); };
     auto writeTF
         = [&](auto path, auto& vecF) { h5Writer.writeTensorFieldAsDataset(h5file, path, vecF); };
 

src/diagnostic/detail/types/meta.hpp

@@ -121,7 +121,7 @@ void MetaDiagnosticWriter<H5Writer>::write(DiagnosticProperties& diagnostic)
         if (tags.count(path) > 0)
         {
             auto& h5 = *fileData_.at(diagnostic.quantity);
-            h5.template write_data_set_flat<GridLayout::dimension>(path + "/tags", tags[path]);
+            h5.write_data_set_flat(path + "/tags", tags[path]);
             tags.erase(path);
         }
     }

src/hdf5/detail/h5/h5_file.hpp

@@ -72,32 +72,32 @@ class HighFiveFile
     NO_DISCARD HiFile& file() { return h5file_; }
 
 
-    template<typename T, std::size_t dim = 1>
-    NO_DISCARD auto read_data_set_flat(std::string path) const
+    template<typename T>
+    NO_DISCARD auto read_data_set_flat(std::string const& path) const
     {
         std::vector<T> data(H5Easy::getSize(h5file_, path));
         h5file_.getDataSet(path).read_raw(data.data());
         return data;
     }
 
     template<typename T, std::size_t dim = 1>
-    NO_DISCARD auto read_data_set(std::string path) const
+    NO_DISCARD auto read_data_set(std::string const& path) const
     {
         auto data = vector_for_dim<T, dim>();
         h5file_.getDataSet(path).read(data);
         return data;
     }
 
 
-    template<std::size_t dim = 1, typename Data>
-    auto& write_data_set(std::string path, Data const& data)
+    template<typename Data>
+    auto& write_data_set(std::string const& path, Data const& data)
     {
         h5file_.getDataSet(path).write(data);
         return *this;
     }
 
-    template<std::size_t dim = 1, typename Data>
-    auto& write_data_set_flat(std::string path, Data const& data)
+    template<typename Data>
+    auto& write_data_set_flat(std::string const& path, Data const& data)
     {
         h5file_.getDataSet(path).write_raw(data);
         return *this;
@@ -188,7 +188,7 @@ class HighFiveFile
      * the same
      */
     template<typename Data>
-    void write_attributes_per_mpi(std::string path, std::string key, Data const& data)
+    void write_attributes_per_mpi(std::string const& path, std::string key, Data const& data)
     {
         constexpr bool data_is_vector = core::is_std_vector_v<Data>;
 
@@ -245,10 +245,10 @@ class HighFiveFile
     }
 
 
-    HighFiveFile(const HighFiveFile&)             = delete;
-    HighFiveFile(const HighFiveFile&&)            = delete;
-    HighFiveFile& operator=(const HighFiveFile&)  = delete;
-    HighFiveFile& operator=(const HighFiveFile&&) = delete;
+    HighFiveFile(HighFiveFile const&)             = delete;
+    HighFiveFile(HighFiveFile const&&)            = delete;
+    HighFiveFile& operator=(HighFiveFile const&)  = delete;
+    HighFiveFile& operator=(HighFiveFile const&&) = delete;
 
 private:
     HighFive::FileAccessProps fapl_;

src/hdf5/writer/particle_writer.hpp

@@ -1,15 +1,14 @@
 #ifndef PHARE_HDF5_PARTICLE_WRITER_HPP
 #define PHARE_HDF5_PARTICLE_WRITER_HPP
 
-#include <array>
-#include <vector>
-
-#include "hdf5/detail/hdf5_utils.hpp"
-#include "hdf5/detail/h5/h5_file.hpp"
 
 #include "core/def.hpp"
 #include "core/data/particles/particle_packer.hpp"
 
+#include "hdf5/detail/hdf5_utils.hpp"
+#include "hdf5/detail/h5/h5_file.hpp"
+
+#include <vector>
 
 namespace PHARE::hdf5
 {
@@ -19,23 +18,29 @@
     template<typename H5File, typename Particles>
     static void write(H5File& h5file, Particles const& particles, std::string const& path)
     {
-        auto constexpr dim = Particles::dimension;
-        using Packer       = core::ParticlePacker<dim>;
+        constexpr auto dim              = Particles::dimension;
+        using Packer                    = core::ParticlePacker<dim>;
+        constexpr auto particle_members = Packer::empty();
+        static auto& keys               = Packer::keys();
+
 
-        Packer packer(particles);
-        core::ContiguousParticles<dim> copy{particles.size()};
-        packer.pack(copy);
+        Packer{particles}.pack_ranges_into([&](auto const& arr, auto const from) {
+            auto const soa_members = arr();
 
-        std::size_t part_idx = 0;
-        core::apply(copy.as_tuple(), [&](auto const& arg) {
-            auto data_path = path + packer.keys()[part_idx++];
-            h5file.template write_data_set_flat<2>(data_path, arg.data());
+            core::for_N<Packer::n_keys>([&](auto ki) {
+                auto const [key, member] = std::get<ki>(soa_members);
+                auto const actual        = std::get<ki>(particle_members);
+
+                h5file.file()
+                    .getDataSet(path + keys[ki])
+                    .select({from, 0ul}, size_for<dim>(actual, arr.size()))
+                    .write_raw(member.data());
+            });
         });
     }
 
 
 
-
     template<std::size_t dim, typename T, typename Size>
     NO_DISCARD auto static size_for(T const& type, Size const& n_particles)
     {

tests/core/data/particles/test_interop.cpp

@@ -55,8 +55,10 @@ TYPED_TEST(ParticleListTest, SoAandAoSInterop)
     EXPECT_EQ(particleArray.size(), size);
     EXPECT_EQ(contiguous.size(), particleArray.size());
 
-    ContiguousParticles<dim> AoSFromSoA{particleArray.size()};
+    ContiguousParticles<dim> AoSFromSoA;
+    EXPECT_EQ(AoSFromSoA.size(), 0);
     ParticlePacker<dim>{particleArray}.pack(AoSFromSoA);
+    EXPECT_EQ(AoSFromSoA.size(), size);
 
     std::size_t i = 0;
     for (auto const& particle : AoSFromSoA)