Use UNDEFINED_EXTENT for this instead of empty extent

Author: franzpoeschel

include/openPMD/Dataset.hpp

@@ -54,9 +54,21 @@ class Dataset
          */
         JOINED_DIMENSION = std::numeric_limits<std::uint64_t>::max(),
         /**
-         * Some backends (i.e. JSON and TOML in template mode) support the
-         * creation of dataset with undefined datatype and extent.
-         * The extent should be given as {UNDEFINED_EXTENT} for that.
+         * In some use cases, the extent needs not be specified.
+         * For these, specify Extent{UNDEFINED_EXTENT}.
+         * Use cases:
+         *
+         * 1. Some backends (i.e. JSON and TOML in template mode) support the
+         *    creation of dataset with undefined datatype and extent.
+         *    The extent should be given as {UNDEFINED_EXTENT} for that.
+         * 2. With openPMD 2.0, the shape of constant components may be omitted
+         *    in writing if it is defined somewhere else as part
+         *    of the same Mesh / Species.
+         *    (https://github.com/openPMD/openPMD-standard/pull/289)
+         *    When reading such datasets, the openPMD-api will try to fill in
+         *    the missing extents, so the extent for constistently-defined
+         *    datasets should ideally not be reported by the read-side API
+         *    as undefined.
          */
         UNDEFINED_EXTENT = std::numeric_limits<std::uint64_t>::max() - 1
     };
@@ -87,5 +99,8 @@ class Dataset
 
     std::optional<size_t> joinedDimension() const;
     static std::optional<size_t> joinedDimension(Extent const &);
+
+    bool undefinedExtent() const;
+    static bool undefinedExtent(Extent const &);
 };
 } // namespace openPMD

include/openPMD/IO/IOTask.hpp

@@ -22,6 +22,7 @@
 
 #include "openPMD/ChunkInfo.hpp"
 #include "openPMD/Dataset.hpp"
+#include "openPMD/Error.hpp"
 #include "openPMD/IterationEncoding.hpp"
 #include "openPMD/Streaming.hpp"
 #include "openPMD/auxiliary/Export.hpp"

include/openPMD/RecordComponent.hpp

@@ -165,6 +165,8 @@ class RecordComponent : public BaseRecordComponent
      * * Shrinking any dimension's extent.
      * * Changing the number of dimensions.
      *
+     * The dataset extent may be empty to indicate undefined extents.
+     *
      * Backend support for resizing datasets:
      * * JSON: Supported
      * * ADIOS2: Supported as of ADIOS2 2.7.0

src/Dataset.cpp

@@ -95,4 +95,13 @@ std::optional<size_t> Dataset::joinedDimension(Extent const &extent)
     }
     return res;
 }
+
+bool Dataset::undefinedExtent() const
+{
+    return undefinedExtent(extent);
+}
+bool Dataset::undefinedExtent(Extent const &e)
+{
+    return e.size() == 1 && e.at(0) == Dataset::UNDEFINED_EXTENT;
+}
 } // namespace openPMD

src/IO/ADIOS/ADIOS2IOHandler.cpp

@@ -801,6 +801,12 @@ void ADIOS2IOHandlerImpl::createDataset(
             "only is not possible.");
     }
 
+    if (Dataset::undefinedExtent(parameters.extent))
+    {
+        throw error::OperationUnsupportedInBackend(
+            "ADIOS2", "No support for Datasets with undefined extent.");
+    }
+
     if (!writable->written)
     {
         /* Sanitize name */
@@ -962,6 +968,12 @@ void ADIOS2IOHandlerImpl::extendDataset(
     VERIFY_ALWAYS(
         access::write(m_handler->m_backendAccess),
         "[ADIOS2] Cannot extend datasets in read-only mode.");
+    if (Dataset::undefinedExtent(parameters.extent))
+    {
+        throw error::OperationUnsupportedInBackend(
+            "ADIOS2", "No support for Datasets with undefined extent.");
+    }
+
     setAndGetFilePosition(writable);
     auto file = refreshFileFromParent(writable, /* preferParentFile = */ false);
     std::string name = nameOfVariable(writable);

src/IO/HDF5/HDF5IOHandler.cpp

@@ -475,6 +475,11 @@ void HDF5IOHandlerImpl::createDataset(
         error::throwOperationUnsupportedInBackend(
             "HDF5", "Joined Arrays currently only supported in ADIOS2");
     }
+    else if (Dataset::undefinedExtent(parameters.extent))
+    {
+        throw error::OperationUnsupportedInBackend(
+            "HDF5", "No support for Datasets with undefined extent.");
+    }
 
     if (!writable->written)
     {
@@ -845,6 +850,11 @@ void HDF5IOHandlerImpl::extendDataset(
         error::throwOperationUnsupportedInBackend(
             "HDF5", "Joined Arrays currently only supported in ADIOS2");
     }
+    else if (Dataset::undefinedExtent(parameters.extent))
+    {
+        throw error::OperationUnsupportedInBackend(
+            "HDF5", "No support for Datasets with undefined extent.");
+    }
 
     auto res = getFile(writable);
     if (!res)

src/RecordComponent.cpp

@@ -73,7 +73,7 @@ or alternatively the JSON option {"verify_homogeneous_extents": false}.
         Attributable const &callsite, RecordComponent &rc)
     {
         auto extent = rc.getExtent();
-        if (extent.empty())
+        if (Dataset::undefinedExtent(extent))
         {
             without_extent.emplace_back(rc);
         }
@@ -216,9 +216,8 @@ RecordComponent &RecordComponent::resetDataset(Dataset d)
         rc.m_hasBeenExtended = true;
     }
 
-    // @todo check this while flushing
-    // if (d.extent.empty())
-    //     throw std::runtime_error("Dataset extent must be at least 1D.");
+    if (d.extent.empty())
+        throw std::runtime_error("Dataset extent must be at least 1D.");
     if (d.empty())
     {
         if (d.extent.empty())
@@ -275,7 +274,7 @@ Extent RecordComponent::getExtent() const
     }
     else
     {
-        return {};
+        return {Dataset::UNDEFINED_EXTENT};
     }
 }
 
@@ -402,7 +401,7 @@ void RecordComponent::flush(
         }
         auto constant_component_write_shape = [&]() {
             auto extent = getExtent();
-            return !extent.empty() &&
+            return !Dataset::undefinedExtent(extent) &&
                 std::none_of(extent.begin(), extent.end(), [](auto val) {
                     return val == Dataset::JOINED_DIMENSION;
                 });
@@ -542,7 +541,7 @@ void RecordComponent::readBase(bool require_unit_si)
         if (!containsAttribute("shape"))
         {
             setWritten(false, Attributable::EnqueueAsynchronously::No);
-            resetDataset(Dataset(dtype, {}));
+            resetDataset(Dataset(dtype, {Dataset::UNDEFINED_EXTENT}));
             setWritten(true, Attributable::EnqueueAsynchronously::No);
 
             return;

test/Files_SerialIO/components_without_extent.cpp

@@ -27,7 +27,8 @@ void particle_offset_without_extent()
             position_comp.storeChunk(std::move(data), {0}, {5});
 
             auto offset_comp = e["positionOffset"][comp_id];
-            offset_comp.resetDataset({openPMD::Datatype::INT, {}});
+            offset_comp.resetDataset(
+                {openPMD::Datatype::INT, {openPMD::Dataset::UNDEFINED_EXTENT}});
             offset_comp.makeConstant(0);
         }
         write.close();
@@ -57,11 +58,13 @@ void particles_without_any_extent()
         for (auto comp_id : {"x", "y", "z"})
         {
             auto position_comp = e["position"][comp_id];
-            position_comp.resetDataset({openPMD::Datatype::INT, {}});
+            position_comp.resetDataset(
+                {openPMD::Datatype::INT, {openPMD::Dataset::UNDEFINED_EXTENT}});
             position_comp.makeConstant(0);
 
             auto offset_comp = e["positionOffset"][comp_id];
-            offset_comp.resetDataset({openPMD::Datatype::INT, {}});
+            offset_comp.resetDataset(
+                {openPMD::Datatype::INT, {openPMD::Dataset::UNDEFINED_EXTENT}});
             offset_comp.makeConstant(0);
         }
         write.close();
@@ -84,7 +87,9 @@ void particles_without_any_extent()
         {
             for (auto const &component : record.second)
             {
-                REQUIRE(component.second.getExtent() == openPMD::Extent{});
+                REQUIRE(
+                    component.second.getExtent() ==
+                    openPMD::Extent{openPMD::Dataset::UNDEFINED_EXTENT});
             }
         }
     }
@@ -152,7 +157,8 @@ void meshes_with_incomplete_extent()
         for (auto comp_id : {"y", "z"})
         {
             auto comp = E[comp_id];
-            comp.resetDataset({openPMD::Datatype::INT, {}});
+            comp.resetDataset(
+                {openPMD::Datatype::INT, {openPMD::Dataset::UNDEFINED_EXTENT}});
             comp.makeConstant(0);
         }
         write.close();
@@ -220,7 +226,9 @@ void meshes_without_any_extent()
         for (auto comp_id : {"x", "y", "z"})
         {
             auto comp = E[comp_id];
-            comp.resetDataset({openPMD::Datatype::FLOAT, {}});
+            comp.resetDataset(
+                {openPMD::Datatype::FLOAT,
+                 {openPMD::Dataset::UNDEFINED_EXTENT}});
             comp.makeConstant<float>(0);
         }
         write.close();
@@ -241,7 +249,9 @@ void meshes_without_any_extent()
         auto E = read.snapshots()[0].meshes["E"];
         for (auto const &component : E)
         {
-            REQUIRE(component.second.getExtent() == openPMD::Extent{});
+            REQUIRE(
+                component.second.getExtent() ==
+                openPMD::Extent{openPMD::Dataset::UNDEFINED_EXTENT});
         }
     }
 }