Support custom hierarchies with attributes

Author: franzpoeschel

include/openPMD/CustomHierarchy.hpp

@@ -20,22 +20,31 @@
  */
 #pragma once
 
+#include "openPMD/IO/AbstractIOHandler.hpp"
 #include "openPMD/backend/Container.hpp"
 
 #include <iostream>
+#include <set>
+#include <string>
 #include <type_traits>
 
 namespace openPMD
 {
 class CustomHierarchy;
 namespace internal
 {
+    struct MeshesParticlesPath
+    {
+        std::set<std::string> paths;
+        [[nodiscard]] bool ignore(std::string const &name) const;
+    };
     using CustomHierarchyData = ContainerData<CustomHierarchy>;
-}
+} // namespace internal
 
 class CustomHierarchy : public Container<CustomHierarchy>
 {
     friend class Iteration;
+    friend class Container<CustomHierarchy>;
 
 private:
     using Container_t = Container<CustomHierarchy>;
@@ -51,6 +60,10 @@ class CustomHierarchy : public Container<CustomHierarchy>
         Container_t::setData(std::move(data));
     }
 
+    void read(internal::MeshesParticlesPath const &);
+
+    void flush(std::string const &path, internal::FlushParams const &) override;
+
 public:
     CustomHierarchy(CustomHierarchy const &other) = default;
     CustomHierarchy(CustomHierarchy &&other) = default;

src/CustomHierarchy.cpp

@@ -20,9 +20,65 @@
  */
 
 #include "openPMD/CustomHierarchy.hpp"
+#include "openPMD/IO/AbstractIOHandler.hpp"
+#include "openPMD/IO/Access.hpp"
+#include "openPMD/IO/IOTask.hpp"
+#include "openPMD/backend/Attributable.hpp"
 
 namespace openPMD
 {
+namespace internal
+{
+    bool MeshesParticlesPath::ignore(const std::string &name) const
+    {
+        return paths.find(name) != paths.end();
+    }
+} // namespace internal
+
 CustomHierarchy::CustomHierarchy() = default;
-CustomHierarchy::CustomHierarchy(NoInit): Container_t(NoInit()) {}
-}
+CustomHierarchy::CustomHierarchy(NoInit) : Container_t(NoInit())
+{}
+
+void CustomHierarchy::read(internal::MeshesParticlesPath const &mpp)
+{
+    /*
+     * Convention for CustomHierarchy::flush and CustomHierarchy::read:
+     * Path is created/opened already at entry point of method, method needs
+     * to create/open path for contained subpaths.
+     */
+    Attributable::readAttributes(ReadMode::FullyReread);
+    Parameter<Operation::LIST_PATHS> pList;
+    IOHandler()->enqueue(IOTask(this, pList));
+    IOHandler()->flush(internal::defaultFlushParams);
+    for (auto const &path : *pList.paths)
+    {
+        if (mpp.ignore(path))
+        {
+            continue;
+        }
+        Parameter<Operation::OPEN_PATH> pOpen;
+        pOpen.path = path;
+        auto subpath = this->operator[](path);
+        IOHandler()->enqueue(IOTask(&subpath, pOpen));
+        subpath.read(mpp);
+    }
+}
+
+void CustomHierarchy::flush(
+    std::string const & /* path */, internal::FlushParams const &flushParams)
+{
+    /*
+     * Convention for CustomHierarchy::flush and CustomHierarchy::read:
+     * Path is created/opened already at entry point of method, method needs
+     * to create/open path for contained subpaths.
+     */
+    Parameter<Operation::CREATE_PATH> pCreate;
+    for (auto &[name, subpath] : *this)
+    {
+        pCreate.path = name;
+        IOHandler()->enqueue(IOTask(this, pCreate));
+        subpath.flush(name, flushParams);
+    }
+    flushAttributes(flushParams);
+}
+} // namespace openPMD

src/Iteration.cpp

@@ -19,6 +19,7 @@
  * If not, see <http://www.gnu.org/licenses/>.
  */
 #include "openPMD/Iteration.hpp"
+#include "openPMD/CustomHierarchy.hpp"
 #include "openPMD/Dataset.hpp"
 #include "openPMD/Datatype.hpp"
 #include "openPMD/Series.hpp"
@@ -290,8 +291,12 @@ void Iteration::flushVariableBased(
     }
 }
 
+/*
+ * @todo move much of this logic to CustomHierarchy::flush
+ */
 void Iteration::flushIteration(internal::FlushParams const &flushParams)
 {
+    CustomHierarchy::flush("", flushParams);
     if (access::readOnly(IOHandler()->m_frontendAccess))
     {
         for (auto &m : meshes)
@@ -388,6 +393,9 @@ void Iteration::readGorVBased(std::string const &groupPath, bool doBeginStep)
     read_impl(groupPath);
 }
 
+/*
+ * @todo move lots of this to CustomHierarchy::read
+ */
 void Iteration::read_impl(std::string const &groupPath)
 {
     Parameter<Operation::OPEN_PATH> pOpen;
@@ -484,6 +492,21 @@ void Iteration::read_impl(std::string const &groupPath)
         hasParticles = s.containsAttribute("particlesPath");
     }
 
+    {
+        internal::MeshesParticlesPath mpp;
+        if (hasMeshes)
+        {
+            mpp.paths.emplace(auxiliary::trim(
+                s.meshesPath(), [](char const &c) { return c == '/'; }));
+        }
+        if (hasParticles)
+        {
+            mpp.paths.emplace(auxiliary::trim(
+                s.particlesPath(), [](char const &c) { return c == '/'; }));
+        }
+        CustomHierarchy::read(mpp);
+    }
+
     if (hasMeshes)
     {
         try

test/CoreTest.cpp

@@ -1,6 +1,7 @@
 // expose private and protected members for invasive testing
 #include "openPMD/Datatype.hpp"
 #include "openPMD/Error.hpp"
+#include "openPMD/IO/Access.hpp"
 #if openPMD_USE_INVASIVE_TESTS
 #define OPENPMD_private public:
 #define OPENPMD_protected public:
@@ -158,6 +159,39 @@ TEST_CASE("attribute_dtype_test", "[core]")
     }
 }
 
+TEST_CASE("custom_hierarchies", "[core]")
+{
+    std::string filePath = "../samples/custom_hierarchies.json";
+    Series write(filePath, Access::CREATE);
+    write.iterations[0];
+    write.close();
+
+    Series read(filePath, Access::READ_ONLY);
+    REQUIRE(read.iterations[0].size() == 0);
+    read.close();
+
+    write = Series(filePath, Access::READ_WRITE);
+    write.iterations[0]["custom"]["hierarchy"];
+    write.iterations[0]["custom"].setAttribute("string", "attribute");
+    write.iterations[0]["custom"]["hierarchy"].setAttribute("number", 3);
+    write.iterations[0]["no_attributes"];
+    write.close();
+
+    read = Series(filePath, Access::READ_ONLY);
+    REQUIRE(read.iterations[0].size() == 2);
+    REQUIRE(read.iterations[0].count("custom") == 1);
+    REQUIRE(read.iterations[0].count("no_attributes") == 1);
+    REQUIRE(
+        read.iterations[0]["custom"]
+            .getAttribute("string")
+            .get<std::string>() == "attribute");
+    REQUIRE(
+        read.iterations[0]["custom"]["hierarchy"]
+            .getAttribute("number")
+            .get<int>() == 3);
+    read.close();
+}
+
 TEST_CASE("myPath", "[core]")
 {
 #if openPMD_USE_INVASIVE_TESTS