diff --git a/CMakeLists.txt b/CMakeLists.txt index 32007025a6..f2fd9ce644 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -390,6 +390,7 @@ include(${openPMD_SOURCE_DIR}/cmake/dependencies/pybind11.cmake) set(CORE_SOURCE src/config.cpp src/ChunkInfo.cpp + src/CustomHierarchy.cpp src/Dataset.cpp src/Datatype.cpp src/Error.cpp @@ -560,6 +561,7 @@ if(openPMD_HAVE_PYTHON) src/binding/python/Attributable.cpp src/binding/python/BaseRecordComponent.cpp src/binding/python/ChunkInfo.cpp + src/binding/python/CustomHierarchy.cpp src/binding/python/Dataset.cpp src/binding/python/Datatype.cpp src/binding/python/Error.cpp @@ -726,6 +728,7 @@ set(openPMD_PYTHON_EXAMPLE_NAMES 11_particle_dataframe 12_span_write 13_write_dynamic_configuration + 14_custom_hierarchy ) if(openPMD_USE_INVASIVE_TESTS) diff --git a/examples/14_custom_hierarchy.py b/examples/14_custom_hierarchy.py new file mode 100755 index 0000000000..b3eff208a9 --- /dev/null +++ b/examples/14_custom_hierarchy.py @@ -0,0 +1,48 @@ +import numpy as np +import openpmd_api as io + + +def main(): + if "bp" in io.file_extensions: + filename = "../samples/custom_hierarchy.bp" + else: + filename = "../samples/custom_hierarchy.json" + s = io.Series(filename, io.Access.create) + it = s.write_iterations()[100] + + # write openPMD part + temp = it.meshes["temperature"] + temp.axis_labels = ["x", "y"] + temp.unit_dimension = {io.Unit_Dimension.T: 1} + temp.position = [0.5, 0.5] + temp.grid_spacing = [1, 1] + temp.grid_global_offset = [0, 0] + temp.reset_dataset(io.Dataset(np.dtype("double"), [5, 5])) + temp[()] = np.zeros((5, 5)) + + # write NeXus part + nxentry = it["Scan"] + nxentry.set_attribute("NX_class", "NXentry") + nxentry.set_attribute("default", "data") + + data = nxentry["data"] + data.set_attribute("NX_class", "NXdata") + data.set_attribute("signal", "counts") + data.set_attribute("axes", ["two_theta"]) + data.set_attribute("two_theta_indices", [0]) + + counts = data.as_container_of_datasets()["counts"] + counts.set_attribute("units", "counts") + counts.set_attribute("long_name", "photodiode counts") + counts.reset_dataset(io.Dataset(np.dtype("int"), [15])) + counts[()] = np.zeros(15, dtype=np.dtype("int")) + + two_theta = data.as_container_of_datasets()["two_theta"] + two_theta.set_attribute("units", "degrees") + two_theta.set_attribute("long_name", "two_theta (degrees)") + two_theta.reset_dataset(io.Dataset(np.dtype("double"), [15])) + two_theta[()] = np.zeros(15) + + +if __name__ == "__main__": + main() diff --git a/include/openPMD/CustomHierarchy.hpp b/include/openPMD/CustomHierarchy.hpp new file mode 100644 index 0000000000..dc23a97e64 --- /dev/null +++ b/include/openPMD/CustomHierarchy.hpp @@ -0,0 +1,256 @@ +/* Copyright 2023 Franz Poeschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ +#pragma once + +#include "openPMD/IO/AbstractIOHandler.hpp" +#include "openPMD/Mesh.hpp" +#include "openPMD/ParticleSpecies.hpp" +#include "openPMD/RecordComponent.hpp" +#include "openPMD/backend/Container.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace openPMD +{ +class CustomHierarchy; +namespace internal +{ + enum class ContainedType + { + Group, + Mesh, + Particle + }; + struct MeshesParticlesPath + { + std::regex meshRegex; + std::set collectNewMeshesPaths; + std::regex particleRegex; + std::set collectNewParticlesPaths; + + /* + * These values decide which path will be returned upon use of the + * shorthand notation s.iterations[0].meshes or .particles. + * + */ + std::string m_defaultMeshesPath = "meshes"; + std::string m_defaultParticlesPath = "particles"; + + explicit MeshesParticlesPath() = default; + MeshesParticlesPath( + std::vector const &meshes, + std::vector const &particles); + MeshesParticlesPath(Series const &); + + [[nodiscard]] ContainedType + determineType(std::vector const &path) const; + [[nodiscard]] bool + isParticleContainer(std::vector const &path) const; + [[nodiscard]] bool + isMeshContainer(std::vector const &path) const; + }; + + struct CustomHierarchyData + : ContainerData + , ContainerData + , ContainerData + , ContainerData + { + explicit CustomHierarchyData(); + + void syncAttributables(); + +#if 0 + inline Container customHierarchiesWrapped() + { + Container res; + res.setData( + {static_cast *>(this), + [](auto const *) {}}); + return res; + } +#endif + inline Container embeddedDatasetsWrapped() + { + Container res; + res.setData( + {static_cast *>(this), + [](auto const *) {}}); + return res; + } + inline Container embeddedMeshesWrapped() + { + Container res; + res.setData( + {static_cast *>(this), + [](auto const *) {}}); + return res; + } + + inline Container embeddedParticlesWrapped() + { + Container res; + res.setData( + {static_cast *>(this), + [](auto const *) {}}); + return res; + } + +#if 0 + inline Container::InternalContainer & + customHierarchiesInternal() + { + return static_cast *>(this) + ->m_container; + } +#endif + inline Container::InternalContainer & + embeddedDatasetsInternal() + { + return static_cast *>(this) + ->m_container; + } + inline Container::InternalContainer &embeddedMeshesInternal() + { + return static_cast *>(this)->m_container; + } + + inline Container::InternalContainer & + embeddedParticlesInternal() + { + return static_cast *>(this) + ->m_container; + } + }; +} // namespace internal + +template +class ConversibleContainer : public Container +{ + template + friend class ConversibleContainer; + +protected: + using Container_t = Container; + using Data_t = internal::CustomHierarchyData; + static_assert( + std::is_base_of_v); + + ConversibleContainer(Attributable::NoInit) + : Container_t(Attributable::NoInit{}) + {} + + std::shared_ptr m_customHierarchyData; + + [[nodiscard]] Data_t &get() + { + return *m_customHierarchyData; + } + [[nodiscard]] Data_t const &get() const + { + return *m_customHierarchyData; + } + + inline void setData(std::shared_ptr data) + { + m_customHierarchyData = data; + Container_t::setData(std::move(data)); + } + +public: + template + auto asContainerOf() -> ConversibleContainer + { + if constexpr ( + std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v) + { + ConversibleContainer res(Attributable::NoInit{}); + res.setData(m_customHierarchyData); + return res; + } + else + { + static_assert( + auxiliary::dependent_false_v, + "[CustomHierarchy::asContainerOf] Type parameter must be " + "one of: CustomHierarchy, RecordComponent, Mesh, " + "ParticleSpecies."); + } + } +}; + +class CustomHierarchy : public ConversibleContainer +{ + friend class Iteration; + friend class Container; + +private: + using Container_t = Container; + using Parent_t = ConversibleContainer; + using Data_t = typename Parent_t::Data_t; + + using EraseStaleMeshes = internal::EraseStaleEntries>; + using EraseStaleParticles = + internal::EraseStaleEntries>; + void readNonscalarMesh(EraseStaleMeshes &map, std::string const &name); + void readScalarMesh(EraseStaleMeshes &map, std::string const &name); + void readParticleSpecies(EraseStaleParticles &map, std::string const &name); + +protected: + CustomHierarchy(); + CustomHierarchy(NoInit); + + void read(internal::MeshesParticlesPath const &); + void read( + internal::MeshesParticlesPath const &, + std::vector ¤tPath); + + void flush_internal( + internal::FlushParams const &, + internal::MeshesParticlesPath &, + std::vector currentPath); + void flush(std::string const &path, internal::FlushParams const &) override; + + /** + * @brief Link with parent. + * + * @param w The Writable representing the parent. + */ + void linkHierarchy(Writable &w) override; + +public: + CustomHierarchy(CustomHierarchy const &other) = default; + CustomHierarchy(CustomHierarchy &&other) = default; + + CustomHierarchy &operator=(CustomHierarchy const &) = default; + CustomHierarchy &operator=(CustomHierarchy &&) = default; +}; +} // namespace openPMD diff --git a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp index 3691055985..ce33a230db 100644 --- a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp +++ b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp @@ -398,7 +398,8 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl // make sure that the given path exists in proper form in // the passed json value - static void ensurePath(nlohmann::json *json, std::string const &path); + static void + ensurePath(nlohmann::json *json, std::string const &path, Access); // In order not to insert the same file name into the data structures // with a new pointer (e.g. when reopening), search for a possibly diff --git a/include/openPMD/Iteration.hpp b/include/openPMD/Iteration.hpp index 43ee1084bb..a79fc1a56d 100644 --- a/include/openPMD/Iteration.hpp +++ b/include/openPMD/Iteration.hpp @@ -20,6 +20,7 @@ */ #pragma once +#include "openPMD/CustomHierarchy.hpp" #include "openPMD/IterationEncoding.hpp" #include "openPMD/Mesh.hpp" #include "openPMD/ParticleSpecies.hpp" @@ -98,7 +99,7 @@ namespace internal BeginStep beginStep = BeginStepTypes::DontBeginStep{}; }; - class IterationData : public AttributableData + class IterationData : public CustomHierarchyData { public: /* @@ -142,10 +143,22 @@ namespace internal * @see * https://github.com/openPMD/openPMD-standard/blob/latest/STANDARD.md#required-attributes-for-the-basepath */ -class Iteration : public Attributable +class Iteration : public CustomHierarchy { - template - friend class Container; +public: + using IterationIndex_t = uint64_t; + + /* + * Some old compilers have trouble with befriending the entire Container + * template here, so we restrict it + * to Container, more is not needed anyway. + * + * E.g. on gcc-7: + * > error: specialization of 'openPMD::Container' + * > after instantiation + * > friend class Container; + */ + friend class Container; friend class Series; friend class internal::AttributableData; template @@ -154,13 +167,17 @@ class Iteration : public Attributable friend class StatefulIterator; friend class StatefulSnapshotsContainer; -public: Iteration(Iteration const &) = default; Iteration(Iteration &&) = default; Iteration &operator=(Iteration const &) = default; Iteration &operator=(Iteration &&) = default; - using IterationIndex_t = uint64_t; + // These use the openPMD Container class mainly for consistency. + // But they are in fact only aliases that don't actually exist + // in the backend. + // Hence meshes.written() and particles.written() will always be false. + Container meshes{}; + Container particles{}; /** * @tparam T Floating point type of user-selected precision (e.g. float, @@ -268,9 +285,6 @@ class Iteration : public Attributable [[deprecated("This attribute is no longer set by the openPMD-api.")]] bool closedByWriter() const; - Container meshes{}; - Container particles{}; // particleSpecies? - virtual ~Iteration() = default; private: @@ -297,14 +311,25 @@ class Iteration : public Attributable inline void setData(std::shared_ptr data) { m_iterationData = std::move(data); - Attributable::setData(m_iterationData); + CustomHierarchy::setData(m_iterationData); } void flushFileBased( std::string const &, IterationIndex_t, internal::FlushParams const &); void flushGroupBased(IterationIndex_t, internal::FlushParams const &); void flushVariableBased(IterationIndex_t, internal::FlushParams const &); - void flush(internal::FlushParams const &); + /* + * Named flushIteration instead of flush to avoid naming + * conflicts with overridden virtual flush from CustomHierarchy + * class. + */ + void flushIteration(internal::FlushParams const &); + + void sync_meshes_and_particles_from_alias_to_subgroups( + internal::MeshesParticlesPath const &); + void sync_meshes_and_particles_from_subgroups_to_alias( + internal::MeshesParticlesPath const &); + void deferParseAccess(internal::DeferredParseAccess); /* * Control flow for runDeferredParseAccess(), readFileBased(), @@ -334,8 +359,6 @@ class Iteration : public Attributable void readGorVBased( std::string const &groupPath, internal::BeginStep const &beginStep); void read_impl(std::string const &groupPath); - void readMeshes(std::string const &meshesPath); - void readParticles(std::string const &particlesPath); /** * Status after beginning an IO step. Currently includes: @@ -416,12 +439,22 @@ class Iteration : public Attributable */ void setStepStatus(StepStatus); + /* + * @brief Check recursively whether this Iteration is dirty. + * It is dirty if any attribute or dataset is read from or written to + * the backend. + * + * @return true If dirty. + * @return false Otherwise. + */ + bool dirtyRecursive() const; + /** * @brief Link with parent. * * @param w The Writable representing the parent. */ - virtual void linkHierarchy(Writable &w); + void linkHierarchy(Writable &w); /** * @brief Access an iteration in read mode that has potentially not been diff --git a/include/openPMD/Mesh.hpp b/include/openPMD/Mesh.hpp index 77ef8b2886..edbd6ff6ff 100644 --- a/include/openPMD/Mesh.hpp +++ b/include/openPMD/Mesh.hpp @@ -41,6 +41,7 @@ class Mesh : public BaseRecord { friend class Container; friend class Iteration; + friend class CustomHierarchy; public: Mesh(Mesh const &) = default; diff --git a/include/openPMD/ParticleSpecies.hpp b/include/openPMD/ParticleSpecies.hpp index af7aa50375..9f454a0ed5 100644 --- a/include/openPMD/ParticleSpecies.hpp +++ b/include/openPMD/ParticleSpecies.hpp @@ -37,6 +37,7 @@ class ParticleSpecies : public Container friend class Iteration; template friend T &internal::makeOwning(T &self, Series); + friend class CustomHierarchy; public: ParticlePatches particlePatches; diff --git a/include/openPMD/RecordComponent.hpp b/include/openPMD/RecordComponent.hpp index ee29a6d7fa..67322a7d36 100644 --- a/include/openPMD/RecordComponent.hpp +++ b/include/openPMD/RecordComponent.hpp @@ -135,6 +135,7 @@ class RecordComponent : public BaseRecordComponent friend class MeshRecordComponent; template friend T &internal::makeOwning(T &self, Series); + friend class CustomHierarchy; public: enum class Allocation @@ -487,8 +488,9 @@ class RecordComponent : public BaseRecordComponent static constexpr char const *const SCALAR = "\vScalar"; protected: - void flush(std::string const &, internal::FlushParams const &); - void read(bool require_unit_si); + void flush( + std::string const &, internal::FlushParams const &, bool set_defaults); + void read(bool read_defaults); private: /** @@ -536,7 +538,7 @@ OPENPMD_protected BaseRecordComponent::setData(m_recordComponentData); } - void readBase(bool require_unit_si); + void readBase(bool read_defaults); template void verifyChunk(Offset const &, Extent const &) const; diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index 0a568ed559..2036ccb298 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -293,6 +293,7 @@ class Series : public Attributable friend class internal::SeriesData; friend class internal::AttributableData; friend class StatefulSnapshotsContainer; + friend class CustomHierarchy; public: explicit Series(); @@ -446,6 +447,7 @@ class Series : public Attributable * basePath. */ std::string meshesPath() const; + std::vector meshesPaths() const; /** Set the path to mesh * records, relative(!) to basePath. @@ -456,6 +458,7 @@ class Series : public Attributable * @return Reference to modified series. */ Series &setMeshesPath(std::string const &meshesPath); + Series &setMeshesPath(std::vector const &meshesPath); /** * @throw no_such_attribute_error If optional attribute is not present. @@ -489,6 +492,7 @@ class Series : public Attributable * basePath. */ std::string particlesPath() const; + std::vector particlesPaths() const; /** Set the path to groups for each particle * species, relative(!) to basePath. @@ -499,6 +503,7 @@ class Series : public Attributable * @return Reference to modified series. */ Series &setParticlesPath(std::string const &particlesPath); + Series &setParticlesPath(std::vector const &particlesPath); /** * @throw no_such_attribute_error If optional attribute is not present. @@ -899,8 +904,6 @@ OPENPMD_private iterations_iterator end, internal::FlushParams const &flushParams, bool flushIOHandler = true); - void flushMeshesPath(); - void flushParticlesPath(); void flushRankTable(); /* Parameter `read_only_this_single_iteration` used for reopening an * Iteration after closing it. diff --git a/include/openPMD/backend/Attributable.hpp b/include/openPMD/backend/Attributable.hpp index 732b2d1b5c..791ee592e4 100644 --- a/include/openPMD/backend/Attributable.hpp +++ b/include/openPMD/backend/Attributable.hpp @@ -50,6 +50,7 @@ class AbstractFilePosition; class Attributable; class Iteration; class Series; +class CustomHierarchy; namespace internal { @@ -59,6 +60,7 @@ namespace internal class SharedAttributableData { friend class openPMD::Attributable; + friend class openPMD::CustomHierarchy; public: SharedAttributableData(AttributableData *); @@ -104,6 +106,7 @@ namespace internal class AttributableData : public std::shared_ptr { friend class openPMD::Attributable; + friend class openPMD::CustomHierarchy; using SharedData_t = std::shared_ptr; @@ -114,6 +117,17 @@ namespace internal AttributableData(AttributableData &&) = delete; virtual ~AttributableData() = default; + inline std::shared_ptr & + asSharedPtrOfAttributable() + { + return *this; + } + inline std::shared_ptr const & + asSharedPtrOfAttributable() const + { + return *this; + } + AttributableData &operator=(AttributableData const &) = delete; AttributableData &operator=(AttributableData &&) = delete; @@ -158,6 +172,7 @@ namespace internal class BaseRecordData; class RecordComponentData; + struct CustomHierarchyData; /* * Internal function to turn a handle into an owning handle that will keep @@ -209,6 +224,8 @@ class Attributable friend T &internal::makeOwning(T &self, Series); friend class StatefulSnapshotsContainer; friend class internal::AttributableData; + friend class CustomHierarchy; + friend struct internal::CustomHierarchyData; protected: // tag for internal constructor diff --git a/include/openPMD/backend/Attribute.hpp b/include/openPMD/backend/Attribute.hpp index 90ae80582b..e0a0e1fa57 100644 --- a/include/openPMD/backend/Attribute.hpp +++ b/include/openPMD/backend/Attribute.hpp @@ -293,6 +293,25 @@ namespace detail } } } + // conversion cast: turn a 1-element vector into a single value + else if constexpr (auxiliary::IsVector_v) + { + if constexpr (std::is_convertible_v) + { + if (pv->size() != 1) + { + return {std::runtime_error( + "getCast: vector to scalar conversion requires " + "single-element vectors")}; + } + return {U(*pv->begin())}; + } + else + { + return {std::runtime_error( + "getCast: no vector to scalar conversion possible.")}; + } + } else { return {std::runtime_error("getCast: no cast possible.")}; diff --git a/include/openPMD/backend/BaseRecord.hpp b/include/openPMD/backend/BaseRecord.hpp index 3eeca4d8d8..13f703a5d6 100644 --- a/include/openPMD/backend/BaseRecord.hpp +++ b/include/openPMD/backend/BaseRecord.hpp @@ -232,6 +232,7 @@ class BaseRecord private: using T_Self = BaseRecord; + friend class CustomHierarchy; friend class Iteration; friend class ParticleSpecies; friend class PatchRecord; diff --git a/include/openPMD/backend/Container.hpp b/include/openPMD/backend/Container.hpp index 8dda5b992b..a3d1a6a934 100644 --- a/include/openPMD/backend/Container.hpp +++ b/include/openPMD/backend/Container.hpp @@ -57,11 +57,26 @@ namespace traits }; } // namespace traits +class CustomHierarchy; + namespace internal { + template + constexpr inline bool isDerivedFromAttributable = + std::is_base_of_v; + + /* + * Opt out from this check due to the recursive definition of + * class CustomHierarchy : public Container{ ... }; + * Cannot check this while CustomHierarchy is still an incomplete type. + */ + template <> + constexpr inline bool isDerivedFromAttributable = true; + class SeriesData; template class EraseStaleEntries; + struct CustomHierarchyData; template < typename T, @@ -103,7 +118,7 @@ template < class Container : virtual public Attributable { static_assert( - std::is_base_of::value, + internal::isDerivedFromAttributable, "Type of container element must be derived from Writable"); friend class Iteration; @@ -114,6 +129,9 @@ class Container : virtual public Attributable template friend class internal::EraseStaleEntries; friend class StatefulIterator; + friend class SeriesIterator; + friend struct internal::CustomHierarchyData; + friend class CustomHierarchy; protected: using ContainerData = internal::ContainerData; diff --git a/include/openPMD/backend/Writable.hpp b/include/openPMD/backend/Writable.hpp index bfb9c67e03..c0ea88ef67 100644 --- a/include/openPMD/backend/Writable.hpp +++ b/include/openPMD/backend/Writable.hpp @@ -105,6 +105,7 @@ class Writable final friend void debug::printDirty(Series const &); friend struct Parameter; friend struct Parameter; + friend class CustomHierarchy; private: Writable(internal::AttributableData *); diff --git a/include/openPMD/binding/python/Common.hpp b/include/openPMD/binding/python/Common.hpp index c72d72ce83..2c09338cc1 100644 --- a/include/openPMD/binding/python/Common.hpp +++ b/include/openPMD/binding/python/Common.hpp @@ -8,6 +8,7 @@ */ #pragma once +#include "openPMD/CustomHierarchy.hpp" #include "openPMD/Iteration.hpp" #include "openPMD/Mesh.hpp" #include "openPMD/ParticlePatches.hpp" @@ -50,6 +51,7 @@ using PyPatchRecordComponentContainer = Container; using PyBaseRecordRecordComponent = BaseRecord; using PyBaseRecordMeshRecordComponent = BaseRecord; using PyBaseRecordPatchRecordComponent = BaseRecord; +using PyCustomHierarchyContainer = Container; PYBIND11_MAKE_OPAQUE(PyIterationContainer) PYBIND11_MAKE_OPAQUE(PyMeshContainer) PYBIND11_MAKE_OPAQUE(PyPartContainer) @@ -61,3 +63,4 @@ PYBIND11_MAKE_OPAQUE(PyMeshRecordComponentContainer) PYBIND11_MAKE_OPAQUE(PyPatchRecordComponentContainer) PYBIND11_MAKE_OPAQUE(PyBaseRecordRecordComponent) PYBIND11_MAKE_OPAQUE(PyBaseRecordPatchRecordComponent) +PYBIND11_MAKE_OPAQUE(PyCustomHierarchyContainer) diff --git a/src/CustomHierarchy.cpp b/src/CustomHierarchy.cpp new file mode 100644 index 0000000000..70a4489f04 --- /dev/null +++ b/src/CustomHierarchy.cpp @@ -0,0 +1,663 @@ +/* Copyright 2023 Franz Poeschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ + +#include "openPMD/CustomHierarchy.hpp" + +#include "openPMD/Dataset.hpp" +#include "openPMD/Error.hpp" +#include "openPMD/IO/AbstractIOHandler.hpp" +#include "openPMD/IO/Access.hpp" +#include "openPMD/IO/IOTask.hpp" +#include "openPMD/Mesh.hpp" +#include "openPMD/ParticleSpecies.hpp" +#include "openPMD/RecordComponent.hpp" +#include "openPMD/Series.hpp" +#include "openPMD/auxiliary/StringManip.hpp" +#include "openPMD/backend/Attributable.hpp" +#include "openPMD/backend/BaseRecord.hpp" +#include "openPMD/backend/MeshRecordComponent.hpp" +#include "openPMD/backend/Writable.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// @todo add handselected choice of [:punct:] characters to this +// using a macro here to make string interpolation simpler +#define OPENPMD_LEGAL_IDENTIFIER_CHARS "[:alnum:]_" +#define OPENPMD_SINGLE_GLOBBING_CHAR "%" +#define OPENPMD_DOUBLE_GLOBBING_CHAR "%%" + +namespace +{ +template +std::string +concatWithSep(Iterator &&begin, Iterator const &end, std::string const &sep) +{ + if (begin == end) + { + return ""; + } + std::stringstream res; + res << *(begin++); + for (; begin != end; ++begin) + { + res << sep << *begin; + } + return res.str(); +} + +std::string +concatWithSep(std::vector const &v, std::string const &sep) +{ + return concatWithSep(v.begin(), v.end(), sep); +} + +// Not specifying std::regex_constants::optimize here, only using it where +// it makes sense to. +constexpr std::regex_constants::syntax_option_type regex_flags = + std::regex_constants::egrep; + +template +void setDefaultMeshesParticlesPath( + std::vector const &meshes, + std::vector const &particles, + OutParam &writeTarget) +{ + std::regex is_default_path_specification( + "[" OPENPMD_LEGAL_IDENTIFIER_CHARS "]+/", + regex_flags | std::regex_constants::optimize); + constexpr char const *default_default_mesh = "meshes"; + constexpr char const *default_default_particle = "particles"; + for (auto [vec, defaultPath, default_default] : + {std::make_tuple( + &meshes, &writeTarget.m_defaultMeshesPath, default_default_mesh), + std::make_tuple( + &particles, + &writeTarget.m_defaultParticlesPath, + default_default_particle)}) + { + bool set_default = true; + /* + * The first eligible path in meshesPath/particlesPath is used as + * the default, "meshes"/"particles" otherwise. + */ + for (auto const &path : *vec) + { + if (std::regex_match(path, is_default_path_specification)) + { + *defaultPath = openPMD::auxiliary::replace_last(path, "/", ""); + set_default = false; + break; + } + } + if (set_default) + { + *defaultPath = default_default; + } + } +} + +bool anyPathRegexMatches( + std::regex const ®ex, std::vector const &path) +{ + std::string pathToMatch = '/' + concatWithSep(path, "/") + '/'; + return std::regex_match(pathToMatch, regex); +} +} // namespace + +namespace openPMD +{ +namespace internal +{ + namespace + { + std::string globToRegexLongForm(std::string const &glob) + { + return auxiliary::replace_all( + auxiliary::replace_all( + glob, + OPENPMD_DOUBLE_GLOBBING_CHAR, + "([" OPENPMD_LEGAL_IDENTIFIER_CHARS "/]*)"), + OPENPMD_SINGLE_GLOBBING_CHAR, + "([" OPENPMD_LEGAL_IDENTIFIER_CHARS "]*)"); + } + + std::string globToRegexShortForm(std::string const &glob) + { + return "[" OPENPMD_LEGAL_IDENTIFIER_CHARS "/]*/" + glob; + } + } // namespace + + MeshesParticlesPath::MeshesParticlesPath( + std::vector const &meshes, + std::vector const &particles) + { + /* + * /group/meshes/E is a mesh if the meshes path contains: + * + * 1) '/group/meshes/' (absolute path to mesh container) + * 2) 'meshes/' (relative path to mesh container) + * + * All this analogously for particles path. + */ + + // regex for detecting option 1) + // e.g. '/path/to/meshes/': The path to the meshes. Mandatory slashes at + // beginning and end, possibly slashes in + // between. Mandatory slash at beginning might + // be replaced with '%%' to enable paths like + // '%%/path/to/meshes'. + // resolves to: `(/|%%)[[:alnum:]_%/]+/` + std::regex is_legal_long_path_specification( + "(/|" OPENPMD_DOUBLE_GLOBBING_CHAR + ")[" OPENPMD_LEGAL_IDENTIFIER_CHARS OPENPMD_SINGLE_GLOBBING_CHAR + "/]+/", + regex_flags | std::regex_constants::optimize); + + // Regex for detecting option 2) + // e.g. 'meshes/': The name without path. One single mandatory slash + // at the end, no slashes otherwise. + // resolves to `[[:alnum:]_]+/` + std::regex is_legal_short_path_specification( + "[" OPENPMD_LEGAL_IDENTIFIER_CHARS "]+/", + regex_flags | std::regex_constants::optimize); + + for (auto [target_regex, vec] : + {std::make_tuple(&this->meshRegex, &meshes), + std::make_tuple(&this->particleRegex, &particles)}) + { + std::stringstream build_regex; + // neutral element: empty language, regex doesn't match anything + build_regex << "(a^)"; + for (auto const &entry : *vec) + { + if (std::regex_match(entry, is_legal_short_path_specification)) + { + build_regex << "|(" << globToRegexShortForm(entry) << ')'; + } + else if (std::regex_match( + entry, is_legal_long_path_specification)) + { + build_regex << "|(" << globToRegexLongForm(entry) << ')'; + } + else + { + std::cerr + << "[WARNING] Not a legal meshes-/particles-path: '" + << entry << "'. Will skip." << std::endl; + } + } + auto regex_string = build_regex.str(); + // std::cout << "Using regex string: " << regex_string << std::endl; + *target_regex = std::regex( + regex_string, regex_flags | std::regex_constants::optimize); + } + setDefaultMeshesParticlesPath(meshes, particles, *this); + } + + ContainedType MeshesParticlesPath::determineType( + std::vector const &path) const + { + if (isMeshContainer(path)) + { + return ContainedType::Mesh; + } + else if (isParticleContainer(path)) + { + return ContainedType::Particle; + } + else + { + return ContainedType::Group; + } + } + + bool MeshesParticlesPath::isParticleContainer( + std::vector const &path) const + { + return anyPathRegexMatches(particleRegex, path); + } + bool MeshesParticlesPath::isMeshContainer( + std::vector const &path) const + { + return anyPathRegexMatches(meshRegex, path); + } + + CustomHierarchyData::CustomHierarchyData() + { + syncAttributables(); + } + + void CustomHierarchyData::syncAttributables() + { + /* + * m_embeddeddatasets and its friends should point to the same instance + * of Attributable. + * Not strictly necessary to do this explicitly due to virtual + * inheritance (all Attributable instances are the same anyway), + * but let's be explicit about this. + */ + for (auto p : std::initializer_list{ + static_cast *>(this), + static_cast *>(this), + static_cast *>(this), + static_cast *>(this)}) + { + p->asSharedPtrOfAttributable() = this->asSharedPtrOfAttributable(); + } + } +} // namespace internal + +// template +// class ConversibleContainer; + +CustomHierarchy::CustomHierarchy() : ConversibleContainer(NoInit{}) +{ + setData(std::make_shared()); +} +CustomHierarchy::CustomHierarchy(NoInit) : ConversibleContainer(NoInit{}) +{} + +void CustomHierarchy::readNonscalarMesh( + EraseStaleMeshes &map, std::string const &mesh_name) +{ + Parameter pOpen; + Parameter aList; + + Mesh &m = map[mesh_name]; + + pOpen.path = mesh_name; + aList.attributes->clear(); + IOHandler()->enqueue(IOTask(&m, pOpen)); + IOHandler()->enqueue(IOTask(&m, aList)); + IOHandler()->flush(internal::defaultFlushParams); + + auto att_begin = aList.attributes->begin(); + auto att_end = aList.attributes->end(); + auto value = std::find(att_begin, att_end, "value"); + auto shape = std::find(att_begin, att_end, "shape"); + if (value != att_end && shape != att_end) + { + MeshRecordComponent &mrc = m; + IOHandler()->enqueue(IOTask(&mrc, pOpen)); + IOHandler()->flush(internal::defaultFlushParams); + mrc.get().m_isConstant = true; + } + try + { + m.read(); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read mesh with name '" << mesh_name + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + map.forget(mesh_name); + } +} + +void CustomHierarchy::readScalarMesh( + EraseStaleMeshes &map, std::string const &mesh_name) +{ + Parameter pOpen; + Parameter pList; + + Parameter dOpen; + Mesh &m = map[mesh_name]; + dOpen.name = mesh_name; + MeshRecordComponent &mrc = m; + IOHandler()->enqueue(IOTask(&mrc, dOpen)); + IOHandler()->flush(internal::defaultFlushParams); + mrc.setWritten(false, Attributable::EnqueueAsynchronously::No); + mrc.resetDataset(Dataset(*dOpen.dtype, *dOpen.extent)); + mrc.setWritten(true, Attributable::EnqueueAsynchronously::No); + try + { + m.read(); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read mesh with name '" << mesh_name + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + map.forget(mesh_name); + } +} + +void CustomHierarchy::readParticleSpecies( + EraseStaleParticles &map, std::string const &species_name) +{ + Parameter pOpen; + Parameter pList; + + ParticleSpecies &p = map[species_name]; + pOpen.path = species_name; + IOHandler()->enqueue(IOTask(&p, pOpen)); + IOHandler()->flush(internal::defaultFlushParams); + try + { + p.read(); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read particle species with name '" << species_name + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + map.forget(species_name); + } +} + +void CustomHierarchy::read(internal::MeshesParticlesPath const &mpp) +{ + std::vector currentPath; + read(mpp, currentPath); +} + +void CustomHierarchy::read( + internal::MeshesParticlesPath const &mpp, + std::vector ¤tPath) +{ + /* + * Convention for CustomHierarchy::flush and CustomHierarchy::read: + * Path is created/opened already at entry point of method, method needs + * to create/open path for contained subpaths. + */ + + Parameter pList; + IOHandler()->enqueue(IOTask(this, pList)); + + Attributable::readAttributes(ReadMode::FullyReread); + Parameter dList; + IOHandler()->enqueue(IOTask(this, dList)); + IOHandler()->flush(internal::defaultFlushParams); + + std::deque constantComponentsPushback; + auto &data = get(); + EraseStaleMeshes meshesMap(data.embeddedMeshesWrapped()); + EraseStaleParticles particlesMap(data.embeddedParticlesWrapped()); + for (auto const &path : *pList.paths) + { + switch (mpp.determineType(currentPath)) + { + case internal::ContainedType::Group: { + Parameter pOpen; + pOpen.path = path; + auto &subpath = this->operator[](path); + IOHandler()->enqueue(IOTask(&subpath, pOpen)); + currentPath.emplace_back(path); + try + { + subpath.read(mpp, currentPath); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read subgroup '" << path << "' at path '" + << myPath().openPMDPath() + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + container().erase(path); + } + currentPath.pop_back(); + if (subpath.size() == 0 && subpath.containsAttribute("shape") && + subpath.containsAttribute("value")) + { + // This is not a group, but a constant record component + // Writable::~Writable() will deal with removing this from the + // backend again. + constantComponentsPushback.push_back(path); + container().erase(path); + } + break; + } + case internal::ContainedType::Mesh: { + try + { + readNonscalarMesh(meshesMap, path); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read mesh at location '" + << myPath().openPMDPath() << "/" << path + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + meshesMap.forget(path); + } + break; + } + case internal::ContainedType::Particle: { + try + { + readParticleSpecies(particlesMap, path); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read particle species at location '" + << myPath().openPMDPath() << "/" << path + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + particlesMap.forget(path); + } + break; + } + } + } + for (auto const &path : *dList.datasets) + { + switch (mpp.determineType(currentPath)) + { + + case internal::ContainedType::Particle: + std::cerr << "[Warning] Dataset found at '" + << (concatWithSep(currentPath, "/") + "/" + path) + << "' inside the particles path. A particle species is " + "always a group, never a dataset. Will parse as a " + "custom dataset. Storing custom datasets inside the " + "particles path is discouraged." + << std::endl; + [[fallthrough]]; + // Group is a bit of an internal misnomer here, it just means that + // it matches neither meshes nor particles path + case internal::ContainedType::Group: { + auto embeddedDatasets = data.embeddedDatasetsWrapped(); + auto &rc = embeddedDatasets[path]; + Parameter dOpen; + dOpen.name = path; + IOHandler()->enqueue(IOTask(&rc, dOpen)); + try + { + IOHandler()->flush(internal::defaultFlushParams); + rc.setWritten(false, Attributable::EnqueueAsynchronously::No); + rc.resetDataset(Dataset(*dOpen.dtype, *dOpen.extent)); + rc.setWritten(true, Attributable::EnqueueAsynchronously::No); + rc.read(/* read_defaults = */ false); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read contained custom dataset '" << path + << "' at path '" << myPath().openPMDPath() + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + embeddedDatasets.erase(path); + } + break; + } + case internal::ContainedType::Mesh: + try + { + readScalarMesh(meshesMap, path); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read scalar mesh at location '" + << myPath().openPMDPath() << "/" << path + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + meshesMap.forget(path); + } + break; + } + } + + for (auto const &path : constantComponentsPushback) + { + auto embeddedDatasets = data.embeddedDatasetsWrapped(); + auto &rc = embeddedDatasets[path]; + try + { + Parameter pOpen; + pOpen.path = path; + IOHandler()->enqueue(IOTask(&rc, pOpen)); + rc.get().m_isConstant = true; + rc.read(/* read_defaults = */ false); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read dataset at location '" + << myPath().openPMDPath() << "/" << path + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + embeddedDatasets.erase(path); + } + } + setDirty(false); +} + +void CustomHierarchy::flush_internal( + internal::FlushParams const &flushParams, + internal::MeshesParticlesPath &mpp, + std::vector currentPath) +{ + /* + * Convention for CustomHierarchy::flush and CustomHierarchy::read: + * Path is created/opened already at entry point of method, method needs + * to create/open path for contained subpaths. + */ + + // No need to do anything in access::readOnly since meshes and particles + // are initialized as aliases for subgroups at parsing time + auto &data = get(); + if (access::write(IOHandler()->m_frontendAccess)) + { + flushAttributes(flushParams); + } + + Parameter pCreate; + for (auto &[name, subpath] : *this) + { + if (!subpath.written()) + { + pCreate.path = name; + IOHandler()->enqueue(IOTask(&subpath, pCreate)); + } + currentPath.emplace_back(name); + subpath.flush_internal(flushParams, mpp, currentPath); + currentPath.pop_back(); + } + for (auto &[name, mesh] : data.embeddedMeshesInternal()) + { + if (!mpp.isMeshContainer(currentPath)) + { + std::string extend_meshes_path; + // Check if this can be covered by shorthand notation + // (e.g. meshesPath == "meshes/") + if (!currentPath.empty() && + *currentPath.rbegin() == mpp.m_defaultMeshesPath) + { + extend_meshes_path = *currentPath.rbegin() + "/"; + } + else + { + // Otherwise use full path + extend_meshes_path = "/" + + (currentPath.empty() + ? "" + : concatWithSep(currentPath, "/") + "/"); + } + mpp.collectNewMeshesPaths.emplace(std::move(extend_meshes_path)); + } + mesh.flush(name, flushParams); + } + for (auto &[name, particleSpecies] : data.embeddedParticlesInternal()) + { + if (!mpp.isParticleContainer(currentPath)) + { + std::string extend_particles_path; + if (!currentPath.empty() && + *currentPath.rbegin() == mpp.m_defaultParticlesPath) + { + // Check if this can be covered by shorthand notation + // (e.g. particlesPath == "particles/") + extend_particles_path = *currentPath.rbegin() + "/"; + } + else + { + // Otherwise use full path + extend_particles_path = "/" + + (currentPath.empty() + ? "" + : concatWithSep(currentPath, "/") + "/"); + ; + } + mpp.collectNewParticlesPaths.emplace( + std::move(extend_particles_path)); + } + particleSpecies.flush(name, flushParams); + } + for (auto &[name, dataset] : get().embeddedDatasetsInternal()) + { + dataset.flush(name, flushParams, /* set_defaults = */ false); + } + + if (flushParams.flushLevel != FlushLevel::SkeletonOnly && + flushParams.flushLevel != FlushLevel::CreateOrOpenFiles) + { + setDirty(false); + } +} + +void CustomHierarchy::flush( + std::string const & /* path */, internal::FlushParams const &) +{ + throw std::runtime_error( + "[CustomHierarchy::flush()] Don't use this method. Flushing should be " + "triggered via Iteration class."); +} + +void CustomHierarchy::linkHierarchy(Writable &w) +{ + Attributable::linkHierarchy(w); +} +} // namespace openPMD + +#undef OPENPMD_LEGAL_IDENTIFIER_CHARS +#undef OPENPMD_SINGLE_GLOBBING_CHAR +#undef OPENPMD_DOUBLE_GLOBBING_CHAR diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 44e862ffe2..e942f7cda0 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -597,13 +597,24 @@ ADIOS2IOHandlerImpl::flush(internal::ParsedFlushParams &flushParams) } } + std::vector sorted; + sorted.reserve(m_dirty.size()); for (auto const &file : m_dirty) { - auto file_data = m_fileData.find(file); + sorted.emplace_back(&file); + } + std::sort( + sorted.begin(), sorted.end(), [](auto const &left, auto const &right) { + return **left <= **right; + }); + + for (auto file : sorted) + { + auto file_data = m_fileData.find(*file); if (file_data == m_fileData.end()) { throw error::Internal( - "[ADIOS2 backend] No associated data found for file'" + *file + + "[ADIOS2 backend] No associated data found for file'" + **file + "'."); } file_data->second->flush( diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index 0a10a6dd9c..1d2d00e562 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -24,6 +24,7 @@ #include "openPMD/Error.hpp" #include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/IO/AbstractIOHandlerImpl.hpp" +#include "openPMD/IO/Access.hpp" #include "openPMD/ThrowError.hpp" #include "openPMD/auxiliary/Filesystem.hpp" #include "openPMD/auxiliary/JSONMatcher.hpp" @@ -569,13 +570,13 @@ void JSONIOHandlerImpl::createPath( auto filepos = setAndGetFilePosition(writable, false); jsonVal = &(*jsonVal)[filepos->id]; - ensurePath(jsonVal, path); + ensurePath(jsonVal, path, m_handler->m_backendAccess); path = filepos->id.to_string() + "/" + path; } else { - ensurePath(jsonVal, path); + ensurePath(jsonVal, path, m_handler->m_backendAccess); } m_dirty.emplace(file); @@ -1049,7 +1050,10 @@ void JSONIOHandlerImpl::openPath( std::make_shared(json::json_pointer(path)); } - ensurePath(j, removeSlashes(parameters.path)); + ensurePath( + j, + removeSlashes(parameters.path), + /* Must not modify j */ Access::READ_ONLY); writable->written = true; } @@ -1955,18 +1959,45 @@ bool JSONIOHandlerImpl::hasKey(nlohmann::json const &j, KeyT &&key) } void JSONIOHandlerImpl::ensurePath( - nlohmann::json *jsonp, std::string const &path) + nlohmann::json *jsonp, std::string const &path, Access access) { auto groups = auxiliary::split(path, "/"); - for (std::string &group : groups) + if (access::readOnly(access)) { - // Enforce a JSON object - // the library will automatically create a list if the first - // key added to it is parseable as an int - jsonp = &(*jsonp)[group]; - if (jsonp->is_null()) + for (std::string const &group : groups) { - *jsonp = nlohmann::json::object(); + if (!jsonp->contains(group)) + { + throw error::ReadError( + error::AffectedObject::Group, + error::Reason::NotFound, + "JSON", + "Required group '" + path + "' not present."); + } + jsonp = &(*jsonp).at(group); + if (!jsonp->is_object()) + { + throw error::ReadError( + error::AffectedObject::Group, + error::Reason::UnexpectedContent, + "JSON", + "Required group '" + path + + "' is present, but not a JSON object."); + } + } + } + else + { + for (std::string const &group : groups) + { + // Enforce a JSON object + // the library will automatically create a list if the first + // key added to it is parseable as an int + jsonp = &(*jsonp)[group]; + if (jsonp->is_null()) + { + *jsonp = nlohmann::json::object(); + } } } } diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 64cfaa39d0..c233c475ea 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -19,6 +19,7 @@ * If not, see . */ #include "openPMD/Iteration.hpp" +#include "openPMD/CustomHierarchy.hpp" #include "openPMD/Dataset.hpp" #include "openPMD/Datatype.hpp" #include "openPMD/Error.hpp" @@ -34,12 +35,11 @@ #include "openPMD/backend/Writable.hpp" #include -#include -#include #include #include #include -#include +#include +#include #include namespace openPMD @@ -47,7 +47,7 @@ namespace openPMD using internal::CloseStatus; using internal::DeferredParseAccess; -Iteration::Iteration() : Attributable(NoInit()) +Iteration::Iteration() : CustomHierarchy(NoInit()) { setData(std::make_shared()); setTime(static_cast(0)); @@ -264,7 +264,7 @@ void Iteration::flushFileBased( case FlushLevel::SkeletonOnly: case FlushLevel::InternalFlush: case FlushLevel::UserFlush: - flush(flushParams); + flushIteration(flushParams); break; } } @@ -287,7 +287,7 @@ void Iteration::flushGroupBased( case FlushLevel::SkeletonOnly: case FlushLevel::InternalFlush: case FlushLevel::UserFlush: - flush(flushParams); + flushIteration(flushParams); break; } } @@ -310,7 +310,7 @@ void Iteration::flushVariableBased( case FlushLevel::SkeletonOnly: case FlushLevel::InternalFlush: case FlushLevel::UserFlush: - flush(flushParams); + flushIteration(flushParams); break; } @@ -336,66 +336,132 @@ void Iteration::flushVariableBased( } } -void Iteration::flush(internal::FlushParams const &flushParams) +void Iteration::flushIteration(internal::FlushParams const &flushParams) { Parameter touch; IOHandler()->enqueue(IOTask(&writable(), touch)); - if (access::readOnly(IOHandler()->m_frontendAccess)) + if (flushParams.flushLevel == FlushLevel::CreateOrOpenFiles) { - for (auto &m : meshes) - m.second.flush(m.first, flushParams); - for (auto &species : particles) - species.second.flush(species.first, flushParams); - setDirty(false); + return; } - else - { - /* Find the root point [Series] of this file, - * meshesPath and particlesPath are stored there */ - Series s = retrieveSeries(); - if (!meshes.empty() || s.containsAttribute("meshesPath")) - { - if (!s.containsAttribute("meshesPath")) - { - s.setMeshesPath("meshes/"); - s.flushMeshesPath(); - } - meshes.flush(s.meshesPath(), flushParams); - for (auto &m : meshes) - m.second.flush(m.first, flushParams); - } - else - { - meshes.setDirty(false); - } + /* + * Convention for CustomHierarchy::flush and CustomHierarchy::read: + * Path is created/opened already at entry point of method, method needs + * to create/open path for contained subpaths. + */ - if (!particles.empty() || s.containsAttribute("particlesPath")) - { - if (!s.containsAttribute("particlesPath")) - { - s.setParticlesPath("particles/"); - s.flushParticlesPath(); - } - particles.flush(s.particlesPath(), flushParams); - for (auto &species : particles) - species.second.flush(species.first, flushParams); - } - else + Series s = retrieveSeries(); + std::vector meshesPaths = s.meshesPaths(), + particlesPaths = s.particlesPaths(); + internal::MeshesParticlesPath mpp(meshesPaths, particlesPaths); + + sync_meshes_and_particles_from_alias_to_subgroups(mpp); + + std::vector currentPath; + CustomHierarchy::flush_internal(flushParams, mpp, currentPath); + + sync_meshes_and_particles_from_subgroups_to_alias(mpp); + + if (!mpp.collectNewMeshesPaths.empty() || + !mpp.collectNewParticlesPaths.empty()) + { + for (auto [newly_added_paths, vec] : + {std::make_pair(&mpp.collectNewMeshesPaths, &meshesPaths), + std::make_pair(&mpp.collectNewParticlesPaths, &particlesPaths)}) { - particles.setDirty(false); + std::transform( + newly_added_paths->begin(), + newly_added_paths->end(), + std::back_inserter(*vec), + [](auto const &pair) { return pair; }); } - - flushAttributes(flushParams); + s.setMeshesPath(meshesPaths); + s.setParticlesPath(particlesPaths); } - if (flushParams.flushLevel != FlushLevel::SkeletonOnly) + + if (flushParams.flushLevel != FlushLevel::SkeletonOnly && + flushParams.flushLevel != FlushLevel::CreateOrOpenFiles) { + if (access::write(IOHandler()->m_frontendAccess)) + { + flushAttributes(flushParams); + } setDirty(false); meshes.setDirty(false); particles.setDirty(false); } } +void Iteration::sync_meshes_and_particles_from_alias_to_subgroups( + internal::MeshesParticlesPath const &mpp) +{ + auto sync_meshes_and_particles = + [this](auto &m_or_p, std::string const &defaultPath) { + using type = + typename std::remove_reference_t::mapped_type; + + if (m_or_p.empty()) + { + return; + } + auto container = (*this)[defaultPath].asContainerOf(); + + for (auto &[name, entry] : m_or_p) + { + if (auxiliary::contains(name, '/')) + { + throw std::runtime_error( + "Unimplemented: Multi-level paths in " + "Iteration::meshes/Iteration::particles"); + } + if (auto it = container.find(name); it != container.end()) + { + if (it->second.m_attri->asSharedPtrOfAttributable() == + entry.m_attri->asSharedPtrOfAttributable()) + { + continue; // has been emplaced previously + } + else + { + throw std::runtime_error("asdfasdfasdfasd"); + } + } + else + { + container.emplace(name, entry); + entry.linkHierarchy(container.writable()); + } + } + }; + + sync_meshes_and_particles(meshes, mpp.m_defaultMeshesPath); + sync_meshes_and_particles(particles, mpp.m_defaultParticlesPath); +} + +void Iteration::sync_meshes_and_particles_from_subgroups_to_alias( + internal::MeshesParticlesPath const &mpp) +{ + auto sync_meshes_and_particles = + [this](auto &m_or_p, std::string const &defaultPath) { + using type = + typename std::remove_reference_t::mapped_type; + auto it = this->find(defaultPath); + if (it == this->end()) + { + return; + } + auto container = it->second.asContainerOf(); + for (auto &[name, entry] : container) + { + m_or_p.emplace(name, entry); + } + }; + + sync_meshes_and_particles(meshes, mpp.m_defaultMeshesPath); + sync_meshes_and_particles(particles, mpp.m_defaultParticlesPath); +} + void Iteration::deferParseAccess(DeferredParseAccess dr) { get().m_deferredParseAccess = @@ -534,68 +600,19 @@ void Iteration::read_impl(std::string const &groupPath) "found " + datatypeToString(Attribute(*aRead.resource).dtype) + ")"); - /* Find the root point [Series] of this file, - * meshesPath and particlesPath are stored there */ Series s = retrieveSeries(); Parameter pList; - auto version = IOHandler()->m_standard; - bool hasMeshes = false; - bool hasParticles = false; - if (version <= OpenpmdStandard::v_1_0_1) - { - IOHandler()->enqueue(IOTask(this, pList)); - IOHandler()->flush(internal::defaultFlushParams); - hasMeshes = std::count( - pList.paths->begin(), - pList.paths->end(), - auxiliary::replace_last(s.meshesPath(), "/", "")) == 1; - hasParticles = - std::count( - pList.paths->begin(), - pList.paths->end(), - auxiliary::replace_last(s.particlesPath(), "/", "")) == 1; - pList.paths->clear(); - } - else - { - hasMeshes = s.containsAttribute("meshesPath"); - hasParticles = s.containsAttribute("particlesPath"); - } + IOHandler()->enqueue(IOTask(this, pList)); - if (hasMeshes) - { - try - { - readMeshes(s.meshesPath()); - } - catch (error::ReadError const &err) - { - std::cerr << "Cannot read meshes in iteration " << groupPath - << " and will skip them due to read error:\n" - << err.what() << std::endl; - meshes.container().clear(); - } - } - meshes.setDirty(false); + // @todo restore compatibility with openPMD 1.0.*: + // hasMeshes <-> meshesPath is defined - if (hasParticles) - { - try - { - readParticles(s.particlesPath()); - } - catch (error::ReadError const &err) - { - std::cerr << "Cannot read particles in iteration " << groupPath - << " and will skip them due to read error:\n" - << err.what() << std::endl; - particles.container().clear(); - } - } - particles.setDirty(false); + internal::MeshesParticlesPath mpp(s.meshesPaths(), s.particlesPaths()); + CustomHierarchy::read(mpp); + + sync_meshes_and_particles_from_subgroups_to_alias(mpp); - readAttributes(ReadMode::FullyReread); #ifdef openPMD_USE_INVASIVE_TESTS if (containsAttribute("__openPMD_internal_fail")) { @@ -612,125 +629,6 @@ void Iteration::read_impl(std::string const &groupPath) #endif } -void Iteration::readMeshes(std::string const &meshesPath) -{ - Parameter pOpen; - Parameter pList; - - pOpen.path = meshesPath; - IOHandler()->enqueue(IOTask(&meshes, pOpen)); - - meshes.readAttributes(ReadMode::FullyReread); - - internal::EraseStaleEntries map{meshes}; - - /* obtain all non-scalar meshes */ - IOHandler()->enqueue(IOTask(&meshes, pList)); - IOHandler()->flush(internal::defaultFlushParams); - - Parameter aList; - for (auto const &mesh_name : *pList.paths) - { - Mesh &m = map[mesh_name]; - pOpen.path = mesh_name; - aList.attributes->clear(); - IOHandler()->enqueue(IOTask(&m, pOpen)); - IOHandler()->enqueue(IOTask(&m, aList)); - IOHandler()->flush(internal::defaultFlushParams); - - auto att_begin = aList.attributes->begin(); - auto att_end = aList.attributes->end(); - auto value = std::find(att_begin, att_end, "value"); - auto shape = std::find(att_begin, att_end, "shape"); - if (value != att_end && shape != att_end) - { - MeshRecordComponent &mrc = m; - IOHandler()->enqueue(IOTask(&mrc, pOpen)); - IOHandler()->flush(internal::defaultFlushParams); - mrc.get().m_isConstant = true; - } - try - { - m.read(); - } - catch (error::ReadError const &err) - { - std::cerr << "Cannot read mesh with name '" << mesh_name - << "' and will skip it due to read error:\n" - << err.what() << std::endl; - map.forget(mesh_name); - } - } - - /* obtain all scalar meshes */ - Parameter dList; - IOHandler()->enqueue(IOTask(&meshes, dList)); - IOHandler()->flush(internal::defaultFlushParams); - - Parameter dOpen; - for (auto const &mesh_name : *dList.datasets) - { - Mesh &m = map[mesh_name]; - dOpen.name = mesh_name; - IOHandler()->enqueue(IOTask(&m, dOpen)); - IOHandler()->flush(internal::defaultFlushParams); - MeshRecordComponent &mrc = m; - IOHandler()->enqueue(IOTask(&mrc, dOpen)); - IOHandler()->flush(internal::defaultFlushParams); - mrc.setWritten(false, Attributable::EnqueueAsynchronously::No); - mrc.resetDataset(Dataset(*dOpen.dtype, *dOpen.extent)); - mrc.setWritten(true, Attributable::EnqueueAsynchronously::No); - try - { - m.read(); - } - catch (error::ReadError const &err) - { - std::cerr << "Cannot read mesh with name '" << mesh_name - << "' and will skip it due to read error:\n" - << err.what() << std::endl; - map.forget(mesh_name); - } - } -} - -void Iteration::readParticles(std::string const &particlesPath) -{ - Parameter pOpen; - Parameter pList; - - pOpen.path = particlesPath; - IOHandler()->enqueue(IOTask(&particles, pOpen)); - - particles.readAttributes(ReadMode::FullyReread); - - /* obtain all particle species */ - pList.paths->clear(); - IOHandler()->enqueue(IOTask(&particles, pList)); - IOHandler()->flush(internal::defaultFlushParams); - - internal::EraseStaleEntries map{particles}; - for (auto const &species_name : *pList.paths) - { - ParticleSpecies &p = map[species_name]; - pOpen.path = species_name; - IOHandler()->enqueue(IOTask(&p, pOpen)); - IOHandler()->flush(internal::defaultFlushParams); - try - { - p.read(); - } - catch (error::ReadError const &err) - { - std::cerr << "Cannot read particle species with name '" - << species_name - << "' and will skip it due to read error:\n" - << err.what() << std::endl; - map.forget(species_name); - } - } -} - auto Iteration::beginStep(bool reread) -> BeginStepStatus { BeginStepStatus res; @@ -904,6 +802,29 @@ void Iteration::setStepStatus(StepStatus status) } } +bool Iteration::dirtyRecursive() const +{ + if (dirty() || CustomHierarchy::dirtyRecursive()) + { + return true; + } + for (auto const &pair : particles) + { + if (!pair.second.written()) + { + return true; + } + } + for (auto const &pair : meshes) + { + if (!pair.second.written()) + { + return true; + } + } + return false; +} + void Iteration::linkHierarchy(Writable &w) { Attributable::linkHierarchy(w); diff --git a/src/ParticlePatches.cpp b/src/ParticlePatches.cpp index 491add8be7..c7fac4b654 100644 --- a/src/ParticlePatches.cpp +++ b/src/ParticlePatches.cpp @@ -100,7 +100,7 @@ void ParticlePatches::read() pr.setDirty(false); try { - prc.PatchRecordComponent::read(/* require_unit_si = */ false); + prc.PatchRecordComponent::read(/* read_defaults = */ false); } catch (error::ReadError const &err) { diff --git a/src/Record.cpp b/src/Record.cpp index 7d41fce5c2..487e773a15 100644 --- a/src/Record.cpp +++ b/src/Record.cpp @@ -56,12 +56,14 @@ void Record::flush_impl( { if (scalar()) { - T_RecordComponent::flush(SCALAR, flushParams); + T_RecordComponent::flush( + SCALAR, flushParams, /* set_defaults = */ true); } else { for (auto &comp : *this) - comp.second.flush(comp.first, flushParams); + comp.second.flush( + comp.first, flushParams, /* set_defaults = */ true); } } else @@ -71,7 +73,7 @@ void Record::flush_impl( if (scalar()) { RecordComponent &rc = *this; - rc.flush(name, flushParams); + rc.flush(name, flushParams, /* set_defaults = */ true); } else { @@ -81,7 +83,8 @@ void Record::flush_impl( for (auto &comp : *this) { comp.second.parent() = getWritable(this); - comp.second.flush(comp.first, flushParams); + comp.second.flush( + comp.first, flushParams, /* set_defaults = */ true); } } } @@ -90,12 +93,14 @@ void Record::flush_impl( if (scalar()) { - T_RecordComponent::flush(name, flushParams); + T_RecordComponent::flush( + name, flushParams, /* set_defaults = */ true); } else { for (auto &comp : *this) - comp.second.flush(comp.first, flushParams); + comp.second.flush( + comp.first, flushParams, /* set_defaults = */ true); } } @@ -110,7 +115,7 @@ void Record::read() /* using operator[] will incorrectly update parent */ try { - T_RecordComponent::read(/* require_unit_si = */ true); + T_RecordComponent::read(/* read_defaults = */ true); } catch (error::ReadError const &err) { @@ -134,7 +139,7 @@ void Record::read() rc.get().m_isConstant = true; try { - rc.read(/* require_unit_si = */ true); + rc.read(/* read_defaults = */ true); } catch (error::ReadError const &err) { @@ -161,7 +166,7 @@ void Record::read() rc.setWritten(true, Attributable::EnqueueAsynchronously::No); try { - rc.read(/* require_unit_si = */ true); + rc.read(/* read_defaults = */ true); } catch (error::ReadError const &err) { diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index d5420ea3b8..806f1fb700 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -237,7 +237,9 @@ bool RecordComponent::empty() const } void RecordComponent::flush( - std::string const &name, internal::FlushParams const &flushParams) + std::string const &name, + internal::FlushParams const &flushParams, + bool set_defaults) { auto &rc = get(); if (flushParams.flushLevel == FlushLevel::SkeletonOnly) @@ -276,7 +278,7 @@ void RecordComponent::flush( "before flushing (see RecordComponent::resetDataset())."); } } - if (!containsAttribute("unitSI")) + if (set_defaults && !containsAttribute("unitSI")) { setUnitSI(1); } @@ -360,9 +362,9 @@ void RecordComponent::flush( } } -void RecordComponent::read(bool require_unit_si) +void RecordComponent::read(bool read_defaults) { - readBase(require_unit_si); + readBase(read_defaults); } namespace @@ -387,7 +389,7 @@ namespace }; } // namespace -void RecordComponent::readBase(bool require_unit_si) +void RecordComponent::readBase(bool read_defaults) { using DT = Datatype; // auto & rc = get(); @@ -435,7 +437,7 @@ void RecordComponent::readBase(bool require_unit_si) readAttributes(ReadMode::FullyReread); - if (require_unit_si) + if (read_defaults) { if (!containsAttribute("unitSI")) { diff --git a/src/Series.cpp b/src/Series.cpp index 72e8740340..4f94b3ec2c 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -191,7 +191,27 @@ Series &Series::setBasePath(std::string const &bp) std::string Series::meshesPath() const { - return getAttribute("meshesPath").get(); + auto res = meshesPaths(); + if (res.empty()) + { + throw no_such_attribute_error("meshesPath"); + } + /* + * @todo: Verify that meshesPath has canonical form + */ + return res.at(0); +} + +std::vector Series::meshesPaths() const +{ + if (containsAttribute("meshesPath")) + { + return getAttribute("meshesPath").get>(); + } + else + { + return {}; + } } Series &Series::setMeshesPath(std::string const &mp) @@ -214,6 +234,23 @@ Series &Series::setMeshesPath(std::string const &mp) setDirty(true); return *this; } +Series &Series::setMeshesPath(std::vector const &mp) +{ + // @todo if already written, then append + switch (mp.size()) + { + case 0: + return *this; + case 1: + setAttribute("meshesPath", *mp.begin()); + break; + default: + setAttribute("meshesPath", mp); + break; + } + setDirty(true); + return *this; +} #if openPMD_HAVE_MPI chunk_assignment::RankMeta Series::rankTable(bool collective) @@ -490,7 +527,27 @@ void Series::flushRankTable() std::string Series::particlesPath() const { - return getAttribute("particlesPath").get(); + auto res = particlesPaths(); + if (res.empty()) + { + throw no_such_attribute_error("particlesPath"); + } + /* + * @todo: Verify that particlesPath has canonical form + */ + return res.at(0); +} + +std::vector Series::particlesPaths() const +{ + if (containsAttribute("particlesPath")) + { + return getAttribute("particlesPath").get>(); + } + else + { + return {}; + } } Series &Series::setParticlesPath(std::string const &pp) @@ -513,6 +570,23 @@ Series &Series::setParticlesPath(std::string const &pp) setDirty(true); return *this; } +Series &Series::setParticlesPath(std::vector const &pp) +{ + // @todo if already written, then append + switch (pp.size()) + { + case 0: + return *this; + case 1: + setAttribute("particlesPath", *pp.begin()); + break; + default: + setAttribute("particlesPath", pp); + break; + } + setDirty(true); + return *this; +} std::string Series::author() const { @@ -1310,7 +1384,7 @@ void Series::flushFileBased( break; case IO::HasBeenOpened: // continue below - it->second.flush(flushParams); + it->second.flushIteration(flushParams); break; } @@ -1333,9 +1407,13 @@ void Series::flushFileBased( case Access::READ_WRITE: case Access::CREATE: case Access::APPEND: { - bool allDirty = dirty(); + bool const allDirty = dirty(); for (auto it = begin; it != end; ++it) { + /* reset the dirty bit for every iteration (i.e. file) + * otherwise only the first iteration will have updates attributes + */ + setDirty(allDirty); // Phase 1 switch (openIterationIfDirty(it->first, it->second)) { @@ -1381,12 +1459,7 @@ void Series::flushFileBased( IOHandler()->enqueue(IOTask(&it->second, std::move(fClose))); it->second.get().m_closed = internal::CloseStatus::Closed; } - /* reset the dirty bit for every iteration (i.e. file) - * otherwise only the first iteration will have updates attributes - */ - setDirty(allDirty); } - setDirty(false); // Phase 3 if (flushIOHandler) @@ -1428,7 +1501,7 @@ void Series::flushGorVBased( series.m_snapshotToStep.at(it->first)}; IOHandler()->enqueue(IOTask(this, std::move(param))); } - it->second.flush(flushParams); + it->second.flushIteration(flushParams); break; } @@ -1529,26 +1602,6 @@ void Series::flushGorVBased( } } -void Series::flushMeshesPath() -{ - Parameter aWrite; - aWrite.name = "meshesPath"; - Attribute a = getAttribute("meshesPath"); - aWrite.resource = a.getResource(); - aWrite.dtype = a.dtype; - IOHandler()->enqueue(IOTask(this, aWrite)); -} - -void Series::flushParticlesPath() -{ - Parameter aWrite; - aWrite.name = "particlesPath"; - Attribute a = getAttribute("particlesPath"); - aWrite.resource = a.getResource(); - aWrite.dtype = a.dtype; - IOHandler()->enqueue(IOTask(this, aWrite)); -} - void Series::readFileBased( std::optional read_only_this_single_iteration) { @@ -2293,7 +2346,8 @@ void Series::readBase() aRead.name = "meshesPath"; IOHandler()->enqueue(IOTask(this, aRead)); IOHandler()->flush(internal::defaultFlushParams); - if (auto val = Attribute(*aRead.resource).getOptional(); + if (auto val = Attribute(*aRead.resource) + .getOptional>(); val.has_value()) { /* allow setting the meshes path after completed IO */ @@ -2331,7 +2385,8 @@ void Series::readBase() aRead.name = "particlesPath"; IOHandler()->enqueue(IOTask(this, aRead)); IOHandler()->flush(internal::defaultFlushParams); - if (auto val = Attribute(*aRead.resource).getOptional(); + if (auto val = Attribute(*aRead.resource) + .getOptional>(); val.has_value()) { /* allow setting the meshes path after completed IO */ diff --git a/src/backend/Attributable.cpp b/src/backend/Attributable.cpp index 4fd2e7edb4..d0dc7bce0a 100644 --- a/src/backend/Attributable.cpp +++ b/src/backend/Attributable.cpp @@ -297,10 +297,10 @@ void Attributable::flushAttributes(internal::FlushParams const &flushParams) } } // Do this outside the if branch to also setDirty to dirtyRecursive - if (flushParams.flushLevel != FlushLevel::SkeletonOnly) - { - setDirty(false); - } + assert( + flushParams.flushLevel != FlushLevel::SkeletonOnly && + flushParams.flushLevel != FlushLevel::CreateOrOpenFiles); + setDirty(false); } void Attributable::readAttributes(ReadMode mode) diff --git a/src/backend/MeshRecordComponent.cpp b/src/backend/MeshRecordComponent.cpp index ed50080757..10bee1cf45 100644 --- a/src/backend/MeshRecordComponent.cpp +++ b/src/backend/MeshRecordComponent.cpp @@ -64,7 +64,7 @@ void MeshRecordComponent::read() "of any floating point type, found " + datatypeToString(Attribute(*aRead.resource).dtype) + ")"); - readBase(/* require_unit_si = */ true); + readBase(/* read_defaults = */ true); } void MeshRecordComponent::flush( @@ -75,7 +75,7 @@ void MeshRecordComponent::flush( { setPosition(std::vector{0}); } - RecordComponent::flush(name, params); + RecordComponent::flush(name, params, /* set_defaults = */ true); } template diff --git a/src/backend/PatchRecord.cpp b/src/backend/PatchRecord.cpp index 5d2b38d50f..47874daae7 100644 --- a/src/backend/PatchRecord.cpp +++ b/src/backend/PatchRecord.cpp @@ -48,10 +48,11 @@ void PatchRecord::flush_impl( path, flushParams); // warning (clang-tidy-10): // bugprone-parent-virtual-call for (auto &comp : *this) - comp.second.flush(comp.first, flushParams); + comp.second.flush( + comp.first, flushParams, /* set_defaults = */ true); } else - T_RecordComponent::flush(path, flushParams); + T_RecordComponent::flush(path, flushParams, /* set_defaults = */ true); if (flushParams.flushLevel != FlushLevel::SkeletonOnly) { setDirty(false); @@ -95,7 +96,7 @@ void PatchRecord::read() prc.setWritten(true, Attributable::EnqueueAsynchronously::No); try { - prc.read(/* require_unit_si = */ false); + prc.read(/* read_defaults = */ false); } catch (error::ReadError const &err) { diff --git a/src/binding/python/Container.cpp b/src/binding/python/Container.cpp new file mode 100644 index 0000000000..d49185e57d --- /dev/null +++ b/src/binding/python/Container.cpp @@ -0,0 +1,71 @@ +/* Copyright 2018-2021 Axel Huebl + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + * + * The function `bind_container` is based on std_bind.h in pybind11 + * Copyright (c) 2016 Sergey Lyskov and Wenzel Jakob + * + * BSD-style license, see pybind11 LICENSE file. + */ + +#include + +#include "openPMD/Iteration.hpp" +#include "openPMD/Mesh.hpp" +#include "openPMD/ParticlePatches.hpp" +#include "openPMD/ParticleSpecies.hpp" +#include "openPMD/Record.hpp" +#include "openPMD/Series.hpp" +#include "openPMD/backend/BaseRecord.hpp" +#include "openPMD/backend/BaseRecordComponent.hpp" +#include "openPMD/backend/Container.hpp" +#include "openPMD/backend/MeshRecordComponent.hpp" +#include "openPMD/backend/PatchRecord.hpp" +#include "openPMD/backend/PatchRecordComponent.hpp" +#include "openPMD/binding/python/Container.hpp" + +#include "openPMD/binding/python/Common.hpp" + +void init_Container(py::module &m) +{ + ::detail::create_and_bind_container( + m, "Iteration_Container"); + ::detail::create_and_bind_container( + m, "Mesh_Container"); + ::detail::create_and_bind_container( + m, "Particle_Container"); + ::detail::create_and_bind_container( + m, "Particle_Patches_Container"); + ::detail::create_and_bind_container( + m, "Record_Container"); + ::detail::create_and_bind_container( + m, "Patch_Record_Container"); + ::detail:: + create_and_bind_container( + m, "Record_Component_Container"); + ::detail:: + create_and_bind_container( + m, "Mesh_Record_Component_Container"); + ::detail::create_and_bind_container< + PyPatchRecordComponentContainer, + Attributable>(m, "Patch_Record_Component_Container"); + ::detail:: + create_and_bind_container( + m, "Custom_Hierarchy_Container"); +} diff --git a/src/binding/python/CustomHierarchy.cpp b/src/binding/python/CustomHierarchy.cpp new file mode 100644 index 0000000000..b893d6a174 --- /dev/null +++ b/src/binding/python/CustomHierarchy.cpp @@ -0,0 +1,52 @@ + + +#include "openPMD/CustomHierarchy.hpp" +#include "openPMD/ParticleSpecies.hpp" +#include "openPMD/RecordComponent.hpp" +#include "openPMD/backend/Attributable.hpp" +#include "openPMD/binding/python/Common.hpp" +#include "openPMD/binding/python/Container.H" +#include + +namespace py = pybind11; +using namespace openPMD; + +template +void define_conversible_container(py::module &m, std::string const &name) +{ + using CC = ConversibleContainer; + py::class_, Attributable>(m, name.c_str()) + .def( + "as_container_of_datasets", + &CC::template asContainerOf) + .def("as_container_of_meshes", &CC::template asContainerOf) + .def( + "as_container_of_particles", + &CC::template asContainerOf) + .def( + "as_container_of_custom_hierarchy", + &CC::template asContainerOf); +} + +void init_CustomHierarchy(py::module &m) +{ + auto py_ch_cont = + declare_container( + m, "Container_CustomHierarchy"); + + define_conversible_container( + m, "ConversibleContainer_CustomHierarchy"); + define_conversible_container( + m, "ConversibleContainer_ParticleSpecies"); + define_conversible_container( + m, "ConversibleContainer_RecordComponent"); + define_conversible_container(m, "ConversibleContainer_Mesh"); + + [[maybe_unused]] py::class_< + CustomHierarchy, + ConversibleContainer, + Container, + Attributable> custom_hierarchy(m, "CustomHierarchy"); + + finalize_container(py_ch_cont); +} diff --git a/src/binding/python/Iteration.cpp b/src/binding/python/Iteration.cpp index cd5fecacb0..4c92062e61 100644 --- a/src/binding/python/Iteration.cpp +++ b/src/binding/python/Iteration.cpp @@ -19,6 +19,7 @@ * If not, see . */ #include "openPMD/Iteration.hpp" +#include "openPMD/CustomHierarchy.hpp" #include "openPMD/backend/Attributable.hpp" #include "openPMD/binding/python/Common.hpp" @@ -40,8 +41,11 @@ void init_Iteration(py::module &m) #define OPENPMD_AVOID_CLANG_FORMAT auto cl = OPENPMD_AVOID_CLANG_FORMAT #undef OPENPMD_AVOID_CLANG_FORMAT - - py::class_(m, "Iteration") + py::class_< + Iteration, + CustomHierarchy, + PyCustomHierarchyContainer, + Attributable>(m, "Iteration") .def(py::init()) .def( diff --git a/src/binding/python/Series.cpp b/src/binding/python/Series.cpp index c46d069185..b7cd7b463e 100644 --- a/src/binding/python/Series.cpp +++ b/src/binding/python/Series.cpp @@ -24,6 +24,7 @@ #include "openPMD/Iteration.hpp" #include "openPMD/IterationEncoding.hpp" #include "openPMD/auxiliary/JSON.hpp" +#include "openPMD/auxiliary/Variant.hpp" #include "openPMD/binding/python/Common.hpp" #include "openPMD/binding/python/Pickle.hpp" #include "openPMD/binding/python/auxiliary.hpp" @@ -33,6 +34,7 @@ #include #include +#include #if openPMD_USE_FILESYSTEM_HEADER #include @@ -419,12 +421,60 @@ this method. &Series::openPMDextension, &Series::setOpenPMDextension) .def_property("base_path", &Series::basePath, &Series::setBasePath) - .def_property( - "meshes_path", &Series::meshesPath, &Series::setMeshesPath) .def("get_rank_table", &Series::rankTable, py::arg("collective")) .def("set_rank_table", &Series::setRankTable, py::arg("my_rank_info")) .def_property( - "particles_path", &Series::particlesPath, &Series::setParticlesPath) + "meshes_path", + [](Series &self) + -> std::variant> { + using res_t = + std::variant>; + auto res = self.meshesPaths(); + if (res.size() == 1) + { + return res_t{std::move(res[0])}; + } + else + { + return res_t{std::move(res)}; + } + }, + [](Series &self, + std::variant> const &arg) + -> Series & { + std::visit( + [&](auto const &arg_resolved) { + self.setMeshesPath(arg_resolved); + }, + arg); + return self; + }) + .def_property( + "particles_path", + [](Series &self) + -> std::variant> { + using res_t = + std::variant>; + auto res = self.particlesPaths(); + if (res.size() == 1) + { + return res_t{std::move(res[0])}; + } + else + { + return res_t{std::move(res)}; + } + }, + [](Series &self, + std::variant> const &arg) + -> Series & { + std::visit( + [&](auto const &arg_resolved) { + self.setParticlesPath(arg_resolved); + }, + arg); + return self; + }) .def_property("author", &Series::author, &Series::setAuthor) .def_property( "machine", @@ -467,8 +517,20 @@ this method. .def("set_openPMD", &Series::setOpenPMD) .def("set_openPMD_extension", &Series::setOpenPMDextension) .def("set_base_path", &Series::setBasePath) - .def("set_meshes_path", &Series::setMeshesPath) - .def("set_particles_path", &Series::setParticlesPath) + .def( + "set_meshes_path", + py::overload_cast(&Series::setMeshesPath)) + .def( + "set_meshes_path", + py::overload_cast const &>( + &Series::setMeshesPath)) + .def( + "set_particles_path", + py::overload_cast const &>( + &Series::setParticlesPath)) + .def( + "set_particles_path", + py::overload_cast(&Series::setParticlesPath)) .def("set_author", &Series::setAuthor) .def("set_date", &Series::setDate) .def("set_iteration_encoding", &Series::setIterationEncoding) diff --git a/src/binding/python/openPMD.cpp b/src/binding/python/openPMD.cpp index fe26bfced8..996208bd55 100644 --- a/src/binding/python/openPMD.cpp +++ b/src/binding/python/openPMD.cpp @@ -37,6 +37,7 @@ void init_Dataset(py::module &); void init_Datatype(py::module &); void init_Error(py::module &); void init_Helper(py::module &); +void init_CustomHierarchy(py::module &); void init_Iteration(py::module &); void init_IterationEncoding(py::module &); void init_Mesh(py::module &); @@ -101,6 +102,7 @@ PYBIND11_MODULE(openpmd_api_cxx, m) init_ParticleSpecies(m); init_Mesh(m); + init_CustomHierarchy(m); init_Iteration(m); init_IterationEncoding(m); init_Series(m); diff --git a/test/CoreTest.cpp b/test/CoreTest.cpp index 92839ed131..b5b32c277f 100644 --- a/test/CoreTest.cpp +++ b/test/CoreTest.cpp @@ -186,6 +186,252 @@ TEST_CASE("attribute_dtype_test", "[core]") } } +TEST_CASE("custom_hierarchies", "[core]") +{ + std::string filePath = "../samples/custom_hierarchies.json"; + Series write(filePath, Access::CREATE); + write.iterations[0]; + write.close(); + + Series read(filePath, Access::READ_ONLY); + REQUIRE(read.iterations[0].size() == 0); + read.close(); + + write = Series(filePath, Access::READ_WRITE); + write.iterations[0]["custom"]["hierarchy"]; + write.iterations[0]["custom"].setAttribute("string", "attribute"); + write.iterations[0]["custom"]["hierarchy"].setAttribute("number", 3); + write.iterations[0]["no_attributes"]; + write.close(); + + read = Series(filePath, Access::READ_ONLY); + REQUIRE(read.iterations[0].size() == 2); + REQUIRE(read.iterations[0].count("custom") == 1); + REQUIRE(read.iterations[0].count("no_attributes") == 1); + REQUIRE(read.iterations[0]["custom"].size() == 1); + REQUIRE(read.iterations[0]["custom"].count("hierarchy") == 1); + REQUIRE(read.iterations[0]["custom"]["hierarchy"].size() == 0); + REQUIRE(read.iterations[0]["no_attributes"].size() == 0); + REQUIRE( + read.iterations[0]["custom"] + .getAttribute("string") + .get() == "attribute"); + REQUIRE( + read.iterations[0]["custom"]["hierarchy"] + .getAttribute("number") + .get() == 3); + read.close(); + + write = Series(filePath, Access::READ_WRITE); + { + write.iterations[0]["custom"]["hierarchy"]; + write.iterations[0]["custom"] + .asContainerOf()["emptyDataset"] + .makeEmpty(Datatype::FLOAT, 3); + write.iterations[0]["custom"]["hierarchy"].setAttribute("number", 3); + write.iterations[0]["no_attributes"]; + auto iteration_level_ds = + write.iterations[0] + .asContainerOf()["iteration_level_dataset"]; + iteration_level_ds.resetDataset({Datatype::INT, {10}}); + std::vector data(10, 5); + iteration_level_ds.storeChunk(data); + + auto meshesViaAlias = write.iterations[0].meshes; + meshesViaAlias["E"]["x"].makeEmpty(2); + write.setMeshesPath(std::vector{"fields/", "%%/meshes/"}); + auto meshesManually = + write.iterations[0]["fields"].asContainerOf(); + REQUIRE(meshesManually.size() == 0); + write.flush(); // Synchronized upon flushing + REQUIRE(meshesManually.contains("E")); + REQUIRE(meshesManually.size() == 1); + meshesManually["B"]["x"].makeEmpty(2); + REQUIRE(meshesViaAlias.size() == 1); + write.flush(); + REQUIRE(meshesViaAlias.contains("B")); + REQUIRE(meshesViaAlias.size() == 2); + + write.setParticlesPath("species"); + auto particlesManually = + write.iterations[0]["species"].asContainerOf(); + particlesManually["e"]["position"]["x"].makeEmpty(1); + auto particlesViaAlias = write.iterations[0].particles; + particlesViaAlias["i"]["position"]["x"].makeEmpty(1); + + write.close(); + } + + read = Series(filePath, Access::READ_ONLY); + { + REQUIRE(read.iterations[0].size() == 4); + REQUIRE(read.iterations[0].count("custom") == 1); + REQUIRE(read.iterations[0].count("no_attributes") == 1); + REQUIRE(read.iterations[0].count("fields") == 1); + REQUIRE(read.iterations[0].count("species") == 1); + REQUIRE(read.iterations[0]["custom"].size() == 1); + REQUIRE(read.iterations[0]["custom"].count("hierarchy") == 1); + REQUIRE(read.iterations[0]["custom"]["hierarchy"].size() == 0); + REQUIRE(read.iterations[0]["no_attributes"].size() == 0); + REQUIRE(read.iterations[0]["fields"].asContainerOf().size() == 2); + REQUIRE( + read.iterations[0]["fields"].asContainerOf().contains("E")); + REQUIRE( + read.iterations[0]["fields"].asContainerOf().contains("B")); + REQUIRE(read.iterations[0].meshes.size() == 2); + REQUIRE(read.iterations[0].meshes.contains("E")); + REQUIRE(read.iterations[0].meshes.contains("B")); + REQUIRE( + read.iterations[0]["species"] + .asContainerOf() + .size() == 2); + REQUIRE(read.iterations[0]["species"] + .asContainerOf() + .contains("e")); + REQUIRE(read.iterations[0]["species"] + .asContainerOf() + .contains("i")); + REQUIRE(read.iterations[0].particles.size() == 2); + REQUIRE(read.iterations[0].particles.contains("e")); + REQUIRE(read.iterations[0].particles.contains("i")); + + REQUIRE( + read.iterations[0].asContainerOf().size() == 1); + REQUIRE( + read.iterations[0]["custom"] + .asContainerOf() + .size() == 1); + REQUIRE( + read.iterations[0]["custom"]["hierarchy"] + .asContainerOf() + .size() == 0); + REQUIRE( + read.iterations[0]["no_attributes"] + .asContainerOf() + .size() == 0); + + auto iteration_level_ds = + read.iterations[0] + .asContainerOf()["iteration_level_dataset"]; + REQUIRE(iteration_level_ds.getDatatype() == Datatype::INT); + REQUIRE(iteration_level_ds.getExtent() == Extent{10}); + auto loaded_chunk = iteration_level_ds.loadChunk(); + iteration_level_ds.seriesFlush(); + for (size_t i = 0; i < 10; ++i) + { + REQUIRE(loaded_chunk.get()[i] == 5); + } + + auto constant_dataset = + read.iterations[0]["custom"] + .asContainerOf()["emptyDataset"]; + REQUIRE(constant_dataset.getDatatype() == Datatype::FLOAT); + REQUIRE(constant_dataset.getExtent() == Extent{0, 0, 0}); + } + read.close(); + + write = Series(filePath, Access::READ_WRITE); + { + std::vector data(10, 3); + + auto E_x = write.iterations[0]["custom_meshes"]["meshes"] + .asContainerOf()["E"]["x"]; + E_x.resetDataset({Datatype::INT, {10}}); + E_x.storeChunk(data, {0}, {10}); + + auto e_pos_x = + write.iterations[0]["custom_particles"]["particles"] + .asContainerOf()["e"]["position"]["x"]; + e_pos_x.resetDataset({Datatype::INT, {10}}); + e_pos_x.storeChunk(data, {0}, {10}); + write.close(); + } + + read = Series(filePath, Access::READ_ONLY); + { + auto it0 = read.iterations[0]; + auto custom_meshes = it0["custom_meshes"]; + REQUIRE(custom_meshes["meshes"].asContainerOf().size() == 1); + REQUIRE( + read.iterations[0]["custom_meshes"]["meshes"] + .asContainerOf() + .count("E") == 1); + auto E_x_loaded = read.iterations[0]["custom_meshes"]["meshes"] + .asContainerOf()["E"]["x"] + .loadChunk(); + REQUIRE( + read.iterations[0]["custom_particles"]["particles"] + .asContainerOf() + .size() == 1); + REQUIRE( + read.iterations[0]["custom_particles"]["particles"] + .asContainerOf() + .count("e") == 1); + auto e_pos_x_loaded = + read.iterations[0]["custom_particles"]["particles"] + .asContainerOf()["e"]["position"]["x"] + .loadChunk(); + read.flush(); + + for (size_t i = 0; i < 10; ++i) + { + REQUIRE(E_x_loaded.get()[i] == 3); + REQUIRE(e_pos_x_loaded.get()[i] == 3); + } + } +} + +TEST_CASE("custom_hierarchies_no_rw", "[core]") +{ + std::string filePath = "../samples/custom_hierarchies_no_rw.json"; + Series write(filePath, Access::CREATE); + write.setMeshesPath(std::vector{"%%/meshes/"}); + write.iterations[0]["custom"]["hierarchy"]; + write.iterations[0]["custom"].setAttribute("string", "attribute"); + write.iterations[0]["custom"]["hierarchy"].setAttribute("number", 3); + write.iterations[0]["no_attributes"]; + + { + write.iterations[0]["custom"]["hierarchy"]; + write.iterations[0]["custom"] + .asContainerOf()["emptyDataset"] + .makeEmpty(Datatype::FLOAT, 3); + write.iterations[0]["custom"]["hierarchy"].setAttribute("number", 3); + write.iterations[0]["no_attributes"]; + auto iteration_level_ds = + write.iterations[0] + .asContainerOf()["iteration_level_dataset"]; + iteration_level_ds.resetDataset({Datatype::INT, {10}}); + std::vector data(10, 5); + iteration_level_ds.storeChunk(data); + write.flush(); + } + + { + std::vector data(10, 3); + + auto E_x = write.iterations[0]["custom_meshes"]["meshes"] + .asContainerOf()["E"]["x"]; + E_x.resetDataset({Datatype::INT, {10}}); + E_x.storeChunk(data, {0}, {10}); + + auto e_pos_x = + write.iterations[0]["custom_particles"]["particles"] + .asContainerOf()["e"]["position"]["x"]; + e_pos_x.resetDataset({Datatype::INT, {10}}); + e_pos_x.storeChunk(data, {0}, {10}); + + auto gnihihi = write.iterations[0]["custom_particles"]["particles"] + .asContainerOf(); + auto dataset = gnihihi["custom_dataset"]; + dataset.resetDataset({Datatype::INT, {10}}); + dataset.storeChunk(std::unique_ptr(new int[10]{}), {0}, {10}); + write.close(); + } + + Series read(filePath, Access::READ_ONLY); +} + TEST_CASE("myPath", "[core]") { #if openPMD_USE_INVASIVE_TESTS @@ -214,7 +460,8 @@ TEST_CASE("myPath", "[core]") recordComponent.template makeConstant(5678); }; - REQUIRE(pathOf(iteration.meshes) == vec_t{"data", "1234", "meshes"}); + // iteration.meshes is only an alias without a path of its own + // REQUIRE(pathOf(iteration.meshes) == vec_t{"data", "1234", "meshes"}); auto scalarMesh = iteration.meshes["e_chargeDensity"]; REQUIRE( @@ -233,7 +480,11 @@ TEST_CASE("myPath", "[core]") pathOf(vectorMeshComponent) == vec_t{"data", "1234", "meshes", "E", "x"}); - REQUIRE(pathOf(iteration.particles) == vec_t{"data", "1234", "particles"}); + // iteration.particles is only an alias without a path of its own + // REQUIRE(pathOf(iteration.particles) == vec_t{"data", "1234", + // "particles"}); REQUIRE( + // pathOf(iteration.particles) == + // vec_t{"iterations", "1234", "particles"}); auto speciesE = iteration.particles["e"]; REQUIRE(pathOf(speciesE) == vec_t{"data", "1234", "particles", "e"}); diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index e4b3dead4f..8ab1cbfc6d 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -2823,9 +2823,17 @@ TEST_CASE("git_hdf5_sample_structure_test", "[serial][hdf5]") REQUIRE( o.iterations[100].meshes.parent() == getWritable(&o.iterations[100])); + REQUIRE( + getWritable( + &o.iterations[100]["fields"].asContainerOf()["E"]) == + getWritable(&o.iterations[100].meshes["E"])); + REQUIRE( + o.iterations[100]["fields"].asContainerOf()["E"].parent() == + &o.iterations[100]["fields"].asContainerOf().writable()); REQUIRE( o.iterations[100].meshes["E"].parent() == - getWritable(&o.iterations[100].meshes)); + // Iteration::meshes is only an alias, this is the actual parent + &o.iterations[100]["fields"].asContainerOf().writable()); REQUIRE( o.iterations[100].meshes["E"]["x"].parent() == getWritable(&o.iterations[100].meshes["E"])); @@ -2835,13 +2843,17 @@ TEST_CASE("git_hdf5_sample_structure_test", "[serial][hdf5]") REQUIRE( o.iterations[100].meshes["E"]["z"].parent() == getWritable(&o.iterations[100].meshes["E"])); + REQUIRE( + getWritable(&o.iterations[100].meshes["rho"]) == + getWritable( + &o.iterations[100]["fields"].asContainerOf()["rho"])); REQUIRE( o.iterations[100].meshes["rho"].parent() == - getWritable(&o.iterations[100].meshes)); + getWritable(&o.iterations[100]["fields"])); REQUIRE( o.iterations[100] .meshes["rho"][MeshRecordComponent::SCALAR] - .parent() == getWritable(&o.iterations[100].meshes)); + .parent() == getWritable(&o.iterations[100]["fields"])); REQUIRE_THROWS_AS( o.iterations[100].meshes["cherries"], std::out_of_range); REQUIRE( @@ -2849,7 +2861,11 @@ TEST_CASE("git_hdf5_sample_structure_test", "[serial][hdf5]") getWritable(&o.iterations[100])); REQUIRE( o.iterations[100].particles["electrons"].parent() == - getWritable(&o.iterations[100].particles)); + getWritable(&o.iterations[100]["particles"])); + REQUIRE( + getWritable(&o.iterations[100].particles["electrons"]) == + getWritable(&o.iterations[100]["particles"] + .asContainerOf()["electrons"])); REQUIRE( o.iterations[100].particles["electrons"]["charge"].parent() == getWritable(&o.iterations[100].particles["electrons"]));