From b4678bcfbbb0f927311407c28f579ebaf29f224b Mon Sep 17 00:00:00 2001 From: guj Date: Thu, 27 Mar 2025 17:21:34 -0700 Subject: [PATCH 1/8] working around an unusal encounter when the joined_dim has actual value "max::size_t - 1" --- include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index 9dee72e02e..1c2aefa744 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -509,6 +509,22 @@ class ADIOS2IOHandlerImpl } else { + if (var.ShapeID() == adios2::ShapeID::JoinedArray) + { + // When you reach here due to a mysterious inconsistency observed from time to time + // e.g. adios2::JoinedDim=(~(size_t)0) which for 64 bits = 18446744073709551615 + // but the particle shape, when printed out, turns out to be + // [18446744073709551614] + // + if (!offset.empty()) + { + throw std::runtime_error( + "[ADIOS2] Offset must be an empty vector in case of joined " + "array."); + } + } + else + { for (unsigned int i = 0; i < actualDim; i++) { if (!(joinedDim.has_value() && *joinedDim == i) && @@ -518,6 +534,7 @@ class ADIOS2IOHandlerImpl "[ADIOS2] Dataset access out of bounds."); } } + }// else } var.SetSelection( From 6309c4dd5634d60b6df9d840a536acf7dd17bfb1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 28 Mar 2025 00:59:01 +0000 Subject: [PATCH 2/8] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp | 44 ++++++++++---------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index 1c2aefa744..410363bf5e 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -509,32 +509,34 @@ class ADIOS2IOHandlerImpl } else { - if (var.ShapeID() == adios2::ShapeID::JoinedArray) - { - // When you reach here due to a mysterious inconsistency observed from time to time - // e.g. adios2::JoinedDim=(~(size_t)0) which for 64 bits = 18446744073709551615 - // but the particle shape, when printed out, turns out to be - // [18446744073709551614] - // - if (!offset.empty()) - { - throw std::runtime_error( - "[ADIOS2] Offset must be an empty vector in case of joined " - "array."); - } - } - else - { - for (unsigned int i = 0; i < actualDim; i++) + if (var.ShapeID() == adios2::ShapeID::JoinedArray) { - if (!(joinedDim.has_value() && *joinedDim == i) && - offset[i] + extent[i] > shape[i]) + // When you reach here due to a mysterious inconsistency + // observed from time to time e.g. + // adios2::JoinedDim=(~(size_t)0) which for 64 bits = + // 18446744073709551615 but the particle shape, when printed + // out, turns out to be [18446744073709551614] + // + if (!offset.empty()) { throw std::runtime_error( - "[ADIOS2] Dataset access out of bounds."); + "[ADIOS2] Offset must be an empty vector in case of " + "joined " + "array."); } } - }// else + else + { + for (unsigned int i = 0; i < actualDim; i++) + { + if (!(joinedDim.has_value() && *joinedDim == i) && + offset[i] + extent[i] > shape[i]) + { + throw std::runtime_error( + "[ADIOS2] Dataset access out of bounds."); + } + } + } // else } var.SetSelection( From 1b3ba34912c0b589bb2e6541f2b162dc2e33254a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 1 Apr 2025 14:51:23 +0200 Subject: [PATCH 3/8] Add test for redundant resetDataset() --- test/ParallelIOTest.cpp | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index e6b960913e..5427b4f797 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -1995,7 +1995,29 @@ void joined_dim(std::string const &ext) patchExtent.store(10); } writeFrom.clear(); + // There seems to be a bug making this flush call necessary, need to fix + it.seriesFlush(); it.close(); + + it = s.writeIterations()[200]; + + // Test issue fixed with + // https://github.com/openPMD/openPMD-api/pull/1740 + + auto bug_dataset = it.particles["flush_multiple_times"]["position"]; + + std::vector buffer(length_of_patch * 2); + std::iota(buffer.begin(), buffer.end(), length_of_patch * 2 * rank); + + bug_dataset.resetDataset({Datatype::INT, {Dataset::JOINED_DIMENSION}}); + bug_dataset.storeChunkRaw(buffer.data(), {}, {length_of_patch}); + it.seriesFlush(); + + bug_dataset.resetDataset({Datatype::INT, {Dataset::JOINED_DIMENSION}}); + bug_dataset.storeChunkRaw( + buffer.data() + length_of_patch, {}, {length_of_patch}); + it.seriesFlush(); + s.close(); } From f4dfc304a4072cc2543df94608447077e4c18528 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 1 Apr 2025 12:00:59 +0200 Subject: [PATCH 4/8] Merge check into above logic --- include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp | 39 +++++++------------- 1 file changed, 14 insertions(+), 25 deletions(-) diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index 410363bf5e..f92bc708b5 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -489,7 +489,8 @@ class ADIOS2IOHandlerImpl } } auto joinedDim = joinedDimension(shape); - if (joinedDim.has_value()) + if (joinedDim.has_value() || + var.ShapeID() == adios2::ShapeID::JoinedArray) { if (!offset.empty()) { @@ -497,6 +498,13 @@ class ADIOS2IOHandlerImpl "[ADIOS2] Offset must be an empty vector in case of joined " "array."); } + if (!joinedDim.has_value()) + { + throw std::runtime_error( + "[ADIOS2] Trying to access a dataset as a non-joined " + "array, but it has previously been configured as a joined " + "array."); + } for (unsigned int i = 0; i < actualDim; i++) { if (*joinedDim != i && extent[i] != shape[i]) @@ -509,35 +517,16 @@ class ADIOS2IOHandlerImpl } else { - if (var.ShapeID() == adios2::ShapeID::JoinedArray) + for (unsigned int i = 0; i < actualDim; i++) { - // When you reach here due to a mysterious inconsistency - // observed from time to time e.g. - // adios2::JoinedDim=(~(size_t)0) which for 64 bits = - // 18446744073709551615 but the particle shape, when printed - // out, turns out to be [18446744073709551614] - // - if (!offset.empty()) + if (!(joinedDim.has_value() && *joinedDim == i) && + offset[i] + extent[i] > shape[i]) { throw std::runtime_error( - "[ADIOS2] Offset must be an empty vector in case of " - "joined " - "array."); + "[ADIOS2] Dataset access out of bounds."); } } - else - { - for (unsigned int i = 0; i < actualDim; i++) - { - if (!(joinedDim.has_value() && *joinedDim == i) && - offset[i] + extent[i] > shape[i]) - { - throw std::runtime_error( - "[ADIOS2] Dataset access out of bounds."); - } - } - } // else - } + } // else var.SetSelection( {adios2::Dims(offset.begin(), offset.end()), From b4af6949cacabd997bb6cc4c8c8105489772e90a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 1 Apr 2025 12:27:00 +0200 Subject: [PATCH 5/8] Better error messages in verifyDataset --- include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp | 38 ++++++++++++++------ include/openPMD/auxiliary/StringManip.hpp | 35 ++++++++++++++++++ src/IO/AbstractIOHandlerImpl.cpp | 25 +------------ 3 files changed, 63 insertions(+), 35 deletions(-) diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index f92bc708b5..384f5819e7 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -34,6 +34,7 @@ #include "openPMD/IterationEncoding.hpp" #include "openPMD/ThrowError.hpp" #include "openPMD/auxiliary/JSON_internal.hpp" +#include "openPMD/auxiliary/StringManip.hpp" #include "openPMD/backend/Writable.hpp" #include "openPMD/config.hpp" #include @@ -489,28 +490,44 @@ class ADIOS2IOHandlerImpl } } auto joinedDim = joinedDimension(shape); + auto make_runtime_error = [&](char const *message) { + std::stringstream s; + s << "[ADIOS2IOHandlerImpl::verifyDataset()] " << message; + s << "\nNote: Variable '" << varName << "' has shape "; + auxiliary::write_vec_to_stream(s, shape) + << ", is accessed from offset "; + auxiliary::write_vec_to_stream(s, offset) << " with extent "; + auxiliary::write_vec_to_stream(s, extent); + if (joinedDim.has_value()) + { + s << " (joined dimension on index " << *joinedDim << ")."; + } + else + { + s << " (no joined dimension)."; + } + return std::runtime_error(s.str()); + }; if (joinedDim.has_value() || var.ShapeID() == adios2::ShapeID::JoinedArray) { if (!offset.empty()) { - throw std::runtime_error( - "[ADIOS2] Offset must be an empty vector in case of joined " - "array."); + throw make_runtime_error( + "Offset must be an empty vector in case of joined array."); } if (!joinedDim.has_value()) { - throw std::runtime_error( - "[ADIOS2] Trying to access a dataset as a non-joined " - "array, but it has previously been configured as a joined " - "array."); + throw make_runtime_error( + "Trying to access a dataset as a non-joined array, but it " + "has previously been array."); } for (unsigned int i = 0; i < actualDim; i++) { if (*joinedDim != i && extent[i] != shape[i]) { - throw std::runtime_error( - "[ADIOS2] store_chunk extent of non-joined dimensions " + throw make_runtime_error( + "store_chunk extent of non-joined dimensions " "must be equivalent to the total extent."); } } @@ -522,8 +539,7 @@ class ADIOS2IOHandlerImpl if (!(joinedDim.has_value() && *joinedDim == i) && offset[i] + extent[i] > shape[i]) { - throw std::runtime_error( - "[ADIOS2] Dataset access out of bounds."); + throw make_runtime_error("Dataset access out of bounds."); } } } // else diff --git a/include/openPMD/auxiliary/StringManip.hpp b/include/openPMD/auxiliary/StringManip.hpp index 36778f205f..eb3799d3be 100644 --- a/include/openPMD/auxiliary/StringManip.hpp +++ b/include/openPMD/auxiliary/StringManip.hpp @@ -242,5 +242,40 @@ namespace auxiliary }); return std::forward(s); } + + template + auto write_vec_to_stream(Stream &&s, Vec const &vec) -> Stream && + { + if (vec.empty()) + { + s << "[]"; + } + else + { + s << '['; + auto it = vec.begin(); + s << *it++; + auto end = vec.end(); + for (; it != end; ++it) + { + s << ", " << *it; + } + s << ']'; + } + return std::forward(s); + } + + template + auto vec_as_string(Vec const &vec) -> std::string + { + if (vec.empty()) + { + return "[]"; + } + else + { + return write_vec_to_stream(std::stringstream(), vec).str(); + } + } } // namespace auxiliary } // namespace openPMD diff --git a/src/IO/AbstractIOHandlerImpl.cpp b/src/IO/AbstractIOHandlerImpl.cpp index 95fb7dfa09..faad2f49a9 100644 --- a/src/IO/AbstractIOHandlerImpl.cpp +++ b/src/IO/AbstractIOHandlerImpl.cpp @@ -24,8 +24,8 @@ #include "openPMD/IO/IOTask.hpp" #include "openPMD/Streaming.hpp" #include "openPMD/auxiliary/Environment.hpp" +#include "openPMD/auxiliary/StringManip.hpp" #include "openPMD/auxiliary/Variant.hpp" -#include "openPMD/backend/Attribute.hpp" #include "openPMD/backend/Writable.hpp" #include @@ -47,29 +47,6 @@ AbstractIOHandlerImpl::AbstractIOHandlerImpl(AbstractIOHandler *handler) namespace { - template - auto vec_as_string(Vec const &vec) -> std::string - { - if (vec.empty()) - { - return "[]"; - } - else - { - std::stringstream res; - res << '['; - auto it = vec.begin(); - res << *it++; - auto end = vec.end(); - for (; it != end; ++it) - { - res << ", " << *it; - } - res << ']'; - return res.str(); - } - } - template struct self_or_invoked { From c7f896a79d57a96423b87db0c13e35801f89a2cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 1 Apr 2025 13:45:30 +0200 Subject: [PATCH 6/8] Add further safety guards to createDataset and extendDataset tasks --- include/openPMD/IO/IOTask.hpp | 19 ++++++++++- include/openPMD/RecordComponent.tpp | 8 ++--- src/IO/ADIOS/ADIOS2IOHandler.cpp | 51 +++++++++++++++++++++++++++++ src/RecordComponent.cpp | 7 ++-- src/Series.cpp | 3 +- src/backend/BaseRecordComponent.cpp | 3 +- 6 files changed, 77 insertions(+), 14 deletions(-) diff --git a/include/openPMD/IO/IOTask.hpp b/include/openPMD/IO/IOTask.hpp index e637df2ede..b109db16d2 100644 --- a/include/openPMD/IO/IOTask.hpp +++ b/include/openPMD/IO/IOTask.hpp @@ -116,6 +116,13 @@ struct OPENPMDAPI_EXPORT AbstractParameter std::string const ¤tBackendName, std::string const &warningMessage); + // Used as a tag in some constructors to make callsites explicitly + // acknowledge that joined dimensions will not be automatically resolved + struct I_dont_want_to_use_joined_dimensions_t + {}; + constexpr static I_dont_want_to_use_joined_dimensions_t + I_dont_want_to_use_joined_dimensions{}; + protected: // avoid object slicing // by allow only child classes to use these things for defining their own @@ -359,7 +366,17 @@ template <> struct OPENPMDAPI_EXPORT Parameter : public AbstractParameter { - Parameter() = default; + Parameter(Dataset const &ds) + : extent(ds.extent) + , dtype(ds.dtype) + , options(ds.options) + , joinedDimension(ds.joinedDimension()) + {} + + // default constructor, but callsites need to explicitly acknowledge that + // joined dimensions will not be automatically configured when using it + Parameter(I_dont_want_to_use_joined_dimensions_t) + {} Parameter(Parameter &&) = default; Parameter(Parameter const &) = default; Parameter &operator=(Parameter &&) = default; diff --git a/include/openPMD/RecordComponent.tpp b/include/openPMD/RecordComponent.tpp index 7beaae8b9d..18fe2d98b1 100644 --- a/include/openPMD/RecordComponent.tpp +++ b/include/openPMD/RecordComponent.tpp @@ -356,18 +356,14 @@ RecordComponent::storeChunk(Offset o, Extent e, F &&createBuffer) if (!written()) { auto &rc = get(); - Parameter dCreate; - dCreate.name = rc.m_name; - dCreate.extent = getExtent(); - dCreate.dtype = getDatatype(); - dCreate.joinedDimension = joinedDimension(); if (!rc.m_dataset.has_value()) { throw error::WrongAPIUsage( "[RecordComponent] Must specify dataset type and extent before " "using storeChunk() (see RecordComponent::resetDataset())."); } - dCreate.options = rc.m_dataset.value().options; + Parameter dCreate(rc.m_dataset.value()); + dCreate.name = rc.m_name; IOHandler()->enqueue(IOTask(this, dCreate)); } Parameter getBufferView; diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index d90fe7c8c7..a622d334aa 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -888,6 +888,57 @@ namespace detail { dims.push_back(ext); } + auto joinedDim = joinedDimension(var.Shape()); + auto make_runtime_error = [&](char const *message) { + std::stringstream s; + s << "[ADIOS2IOHandlerImpl::extendDataset()] " << message + << "\nNote: Variable '" << variable << "' has old shape "; + auxiliary::write_vec_to_stream(s, var.Shape()); + if (joinedDim.has_value()) + { + s << " (joined dimension on index " << *joinedDim << ")."; + } + else + { + s << " (no joined dimension)"; + } + s << " and is extended to new shape "; + auxiliary::write_vec_to_stream(s, newShape) << "."; + return std::runtime_error(s.str()); + }; + if (joinedDim.has_value() || + var.ShapeID() == adios2::ShapeID::JoinedArray) + { + if (!joinedDim.has_value()) + { + throw make_runtime_error( + "Inconsistent state of variable: Has shape ID " + "JoinedArray, but its shape contains no value " + "adios2::JoinedDim."); + } + if (newShape.at(*joinedDim) != Dataset::JOINED_DIMENSION) + { + throw make_runtime_error( + "Variable was previously configured with a joined " + "dimension, so the new dataset extent must keep the " + "joined dimension on that index."); + } + dims[*joinedDim] = adios2::JoinedDim; + } + else + { + for (auto s : newShape) + { + if (s == Dataset::JOINED_DIMENSION) + { + throw make_runtime_error( + "Variable was not previously configured with a " + "joined dimension, but is now requested to change " + "extent to a joined array."); + } + } + } + var.SetShape(dims); } diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index fc17909fc6..7657fd16cf 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -312,12 +312,9 @@ void RecordComponent::flush( } else { - Parameter dCreate; + Parameter dCreate( + rc.m_dataset.value()); dCreate.name = name; - dCreate.extent = getExtent(); - dCreate.dtype = getDatatype(); - dCreate.options = rc.m_dataset.value().options; - dCreate.joinedDimension = joinedDimension(); IOHandler()->enqueue(IOTask(this, dCreate)); } } diff --git a/src/Series.cpp b/src/Series.cpp index 47a642a88d..da443018e0 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -415,7 +415,8 @@ void Series::flushRankTable() { return; } - Parameter param; + Parameter param( + AbstractParameter::I_dont_want_to_use_joined_dimensions); param.name = "rankTable"; param.dtype = Datatype::CHAR; param.extent = {uint64_t(size), uint64_t(maxSize)}; diff --git a/src/backend/BaseRecordComponent.cpp b/src/backend/BaseRecordComponent.cpp index 3f0f1b35c0..2cd4551536 100644 --- a/src/backend/BaseRecordComponent.cpp +++ b/src/backend/BaseRecordComponent.cpp @@ -21,6 +21,7 @@ #include "openPMD/backend/BaseRecordComponent.hpp" #include "openPMD/Error.hpp" #include "openPMD/Iteration.hpp" +#include namespace openPMD { @@ -75,7 +76,7 @@ std::optional BaseRecordComponent::joinedDimension() const } else { - return false; + return std::nullopt; } } From 240e25ff1467bd38cf66e8fc8121c164606422fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 2 Apr 2025 12:11:59 +0200 Subject: [PATCH 7/8] Move joinedDim logic into middle-end for extendDataset --- include/openPMD/Dataset.hpp | 1 + include/openPMD/IO/IOTask.hpp | 11 +++++++- src/Dataset.cpp | 5 ++++ src/IO/ADIOS/ADIOS2IOHandler.cpp | 45 ++++++++++++++++++------------- src/IO/HDF5/HDF5IOHandler.cpp | 6 +++++ src/IO/JSON/JSONIOHandlerImpl.cpp | 7 +++++ src/RecordComponent.cpp | 4 +-- src/binding/python/Dataset.cpp | 3 ++- 8 files changed, 60 insertions(+), 22 deletions(-) diff --git a/include/openPMD/Dataset.hpp b/include/openPMD/Dataset.hpp index d79380105a..80513683f9 100644 --- a/include/openPMD/Dataset.hpp +++ b/include/openPMD/Dataset.hpp @@ -86,5 +86,6 @@ class Dataset bool empty() const; std::optional joinedDimension() const; + static std::optional joinedDimension(Extent const &); }; } // namespace openPMD diff --git a/include/openPMD/IO/IOTask.hpp b/include/openPMD/IO/IOTask.hpp index b109db16d2..5dd3891ad8 100644 --- a/include/openPMD/IO/IOTask.hpp +++ b/include/openPMD/IO/IOTask.hpp @@ -405,7 +405,15 @@ template <> struct OPENPMDAPI_EXPORT Parameter : public AbstractParameter { - Parameter() = default; + Parameter(Extent e) : joinedDimension(Dataset::joinedDimension(e)) + { + this->extent = std::move(e); + } + + // default constructor, but callsites need to explicitly acknowledge that + // joined dimensions will not be automatically configured when using it + Parameter(I_dont_want_to_use_joined_dimensions_t) + {} Parameter(Parameter &&) = default; Parameter(Parameter const &) = default; Parameter &operator=(Parameter &&) = default; @@ -418,6 +426,7 @@ struct OPENPMDAPI_EXPORT Parameter } Extent extent = {}; + std::optional joinedDimension; }; template <> diff --git a/src/Dataset.cpp b/src/Dataset.cpp index c1546e9ef0..a56c566805 100644 --- a/src/Dataset.cpp +++ b/src/Dataset.cpp @@ -68,6 +68,11 @@ bool Dataset::empty() const } std::optional Dataset::joinedDimension() const +{ + return joinedDimension(extent); +} + +std::optional Dataset::joinedDimension(Extent const &extent) { std::optional res; for (size_t i = 0; i < extent.size(); ++i) diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index a622d334aa..191729557a 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -873,7 +873,10 @@ namespace detail { template static void call( - adios2::IO &IO, std::string const &variable, Extent const &newShape) + adios2::IO &IO, + std::string const &variable, + Extent const &newShape, + std::optional const &newJoinedDim) { auto var = IO.InquireVariable(variable); if (!var) @@ -888,54 +891,60 @@ namespace detail { dims.push_back(ext); } - auto joinedDim = joinedDimension(var.Shape()); + auto oldJoinedDim = joinedDimension(var.Shape()); auto make_runtime_error = [&](char const *message) { std::stringstream s; s << "[ADIOS2IOHandlerImpl::extendDataset()] " << message << "\nNote: Variable '" << variable << "' has old shape "; auxiliary::write_vec_to_stream(s, var.Shape()); - if (joinedDim.has_value()) + if (oldJoinedDim.has_value()) { - s << " (joined dimension on index " << *joinedDim << ")."; + s << " (joined dimension on index " << *oldJoinedDim << ")"; } else { s << " (no joined dimension)"; } s << " and is extended to new shape "; - auxiliary::write_vec_to_stream(s, newShape) << "."; + auxiliary::write_vec_to_stream(s, newShape); + if (newJoinedDim.has_value()) + { + s << " (joined dimension on index " << *newJoinedDim + << ")."; + } + else + { + s << " (no joined dimension)."; + } return std::runtime_error(s.str()); }; - if (joinedDim.has_value() || + if (oldJoinedDim.has_value() || var.ShapeID() == adios2::ShapeID::JoinedArray) { - if (!joinedDim.has_value()) + if (!oldJoinedDim.has_value()) { throw make_runtime_error( "Inconsistent state of variable: Has shape ID " "JoinedArray, but its shape contains no value " "adios2::JoinedDim."); } - if (newShape.at(*joinedDim) != Dataset::JOINED_DIMENSION) + if (newJoinedDim != oldJoinedDim) { throw make_runtime_error( "Variable was previously configured with a joined " "dimension, so the new dataset extent must keep the " "joined dimension on that index."); } - dims[*joinedDim] = adios2::JoinedDim; + dims[*newJoinedDim] = adios2::JoinedDim; } else { - for (auto s : newShape) + if (newJoinedDim.has_value()) { - if (s == Dataset::JOINED_DIMENSION) - { - throw make_runtime_error( - "Variable was not previously configured with a " - "joined dimension, but is now requested to change " - "extent to a joined array."); - } + throw make_runtime_error( + "Variable was not previously configured with a " + "joined dimension, but is now requested to change " + "extent to a joined array."); } } @@ -958,7 +967,7 @@ void ADIOS2IOHandlerImpl::extendDataset( auto &filedata = getFileData(file, IfFileNotOpen::ThrowError); Datatype dt = detail::fromADIOS2Type(filedata.m_IO.VariableType(name)); switchAdios2VariableType( - dt, filedata.m_IO, name, parameters.extent); + dt, filedata.m_IO, name, parameters.extent, parameters.joinedDimension); } void ADIOS2IOHandlerImpl::openFile( diff --git a/src/IO/HDF5/HDF5IOHandler.cpp b/src/IO/HDF5/HDF5IOHandler.cpp index f852049463..a66cc42703 100644 --- a/src/IO/HDF5/HDF5IOHandler.cpp +++ b/src/IO/HDF5/HDF5IOHandler.cpp @@ -840,6 +840,12 @@ void HDF5IOHandlerImpl::extendDataset( throw std::runtime_error( "[HDF5] Extending an unwritten Dataset is not possible."); + if (parameters.joinedDimension.has_value()) + { + error::throwOperationUnsupportedInBackend( + "HDF5", "Joined Arrays currently only supported in ADIOS2"); + } + auto res = getFile(writable); if (!res) res = getFile(writable->parent); diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index fb33d1cd29..9df7036156 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -711,6 +711,13 @@ void JSONIOHandlerImpl::extendDataset( VERIFY_ALWAYS( access::write(m_handler->m_backendAccess), "[JSON] Cannot extend a dataset in read-only mode.") + + if (parameters.joinedDimension.has_value()) + { + error::throwOperationUnsupportedInBackend( + "JSON", "Joined Arrays currently only supported in ADIOS2"); + } + setAndGetFilePosition(writable); refreshFileFromParent(writable); auto &j = obtainJsonContents(writable); diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index 7657fd16cf..d5420ea3b8 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -339,8 +339,8 @@ void RecordComponent::flush( } else { - Parameter pExtend; - pExtend.extent = rc.m_dataset.value().extent; + Parameter pExtend( + rc.m_dataset.value().extent); IOHandler()->enqueue(IOTask(this, std::move(pExtend))); rc.m_hasBeenExtended = false; } diff --git a/src/binding/python/Dataset.cpp b/src/binding/python/Dataset.cpp index 70d85721f2..313daee63e 100644 --- a/src/binding/python/Dataset.cpp +++ b/src/binding/python/Dataset.cpp @@ -79,7 +79,8 @@ void init_Dataset(py::module &m) }) .def_property_readonly( - "joined_dimension", &Dataset::joinedDimension) + "joined_dimension", + py::overload_cast<>(&Dataset::joinedDimension, py::const_)) .def_readonly("extent", &Dataset::extent) .def("extend", &Dataset::extend) .def_readonly("rank", &Dataset::rank) From 154ce6514e777dbbad980b71fb70db466696335d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 3 Apr 2025 13:47:32 +0200 Subject: [PATCH 8/8] Update include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp --- include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index 384f5819e7..45980776e4 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -542,7 +542,7 @@ class ADIOS2IOHandlerImpl throw make_runtime_error("Dataset access out of bounds."); } } - } // else + } var.SetSelection( {adios2::Dims(offset.begin(), offset.end()),