From 96964f2c3aba8bab887283d493fa1f359a706681 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Mon, 13 Apr 2026 16:28:10 -0700 Subject: [PATCH 1/8] Add PointIndexCodec Signed-off-by: Dan Bailey --- openvdb/openvdb/CMakeLists.txt | 1 + openvdb/openvdb/codecs/PointIndexCodec.h | 157 +++++++++++++++++++++ openvdb/openvdb/io/Codec.cc | 2 + openvdb/openvdb/unittest/CMakeLists.txt | 1 + openvdb/openvdb/unittest/TestPointCodec.cc | 147 +++++++++++++++++++ 5 files changed, 308 insertions(+) create mode 100644 openvdb/openvdb/codecs/PointIndexCodec.h create mode 100644 openvdb/openvdb/unittest/TestPointCodec.cc diff --git a/openvdb/openvdb/CMakeLists.txt b/openvdb/openvdb/CMakeLists.txt index 3572583e20..e9437b86de 100644 --- a/openvdb/openvdb/CMakeLists.txt +++ b/openvdb/openvdb/CMakeLists.txt @@ -367,6 +367,7 @@ set(OPENVDB_LIBRARY_INCLUDE_FILES set(OPENVDB_LIBRARY_CODECS_INCLUDE_FILES codecs/BoolCodec.h + codecs/PointIndexCodec.h codecs/ScalarCodec.h codecs/TopologyCodec.h codecs/ValueMaskCodec.h diff --git a/openvdb/openvdb/codecs/PointIndexCodec.h b/openvdb/openvdb/codecs/PointIndexCodec.h new file mode 100644 index 0000000000..d8d9f1ee32 --- /dev/null +++ b/openvdb/openvdb/codecs/PointIndexCodec.h @@ -0,0 +1,157 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 + +#ifndef OPENVDB_IO_CODECS_POINTINDEXCODEC_HAS_BEEN_INCLUDED +#define OPENVDB_IO_CODECS_POINTINDEXCODEC_HAS_BEEN_INCLUDED + +#include + +#include + +#include "ScalarLeafCodec.h" +#include "TopologyCodec.h" + +namespace openvdb { +OPENVDB_USE_VERSION_NAMESPACE +namespace OPENVDB_VERSION_NAME { +namespace codecs { +namespace internal { + +template +struct ReadPointIndexBuffersOp +{ + using TreeT = typename GridT::TreeType; + using RootT = typename TreeT::RootNodeType; + using LeafT = typename TreeT::LeafNodeType; + using ValueT = typename TreeT::ValueType; + + ReadPointIndexBuffersOp(std::istream& _is, bool _saveFloatAsHalf, + const ValueT& _background) + : is(_is) + , saveFloatAsHalf(_saveFloatAsHalf) + , background(_background) { } + + template + void operator()(NodeT&, size_t) { } + + void operator()(LeafT& leaf, size_t) + { + using BaseLeaf = typename LeafT::BaseLeaf; + + // Read the value mask and voxel data via base class + BaseLeaf& baseLeaf = static_cast(leaf); + readScalarLeafBuffers(baseLeaf, is, saveFloatAsHalf, background, /*skip=*/false, /*clipBBox=*/nullptr); + + // Read the number of indices. + Index64 numIndices = Index64(0); + is.read(reinterpret_cast(&numIndices), sizeof(Index64)); + + // Read the indices data. + leaf.indices().resize(size_t(numIndices)); + is.read(reinterpret_cast(leaf.indices().data()), numIndices * sizeof(ValueT)); + + // Reserved for future use. + Index64 auxDataBytes = Index64(0); + is.read(reinterpret_cast(&auxDataBytes), sizeof(Index64)); + if (auxDataBytes > 0) { + // For now, read and discard any auxiliary data. + std::unique_ptr auxData{new char[auxDataBytes]}; + is.read(auxData.get(), auxDataBytes); + } + } + + std::istream& is; + const bool saveFloatAsHalf; + const ValueT& background; +}; // struct ReadPointIndexBuffersOp + +template +struct WritePointIndexBuffersOp +{ + using TreeT = typename GridT::TreeType; + using LeafT = typename TreeT::LeafNodeType; + using ValueT = typename TreeT::ValueType; + + WritePointIndexBuffersOp(std::ostream& _os, bool _saveFloatAsHalf) + : os(_os) + , saveFloatAsHalf(_saveFloatAsHalf) { } + + template + void operator()(const NodeT&, size_t) { } + + void operator()(const LeafT& leaf, size_t) + { + using BaseLeaf = typename LeafT::BaseLeaf; + + // Write out the value mask and voxel values via base class + const BaseLeaf& baseLeaf = static_cast(leaf); + writeScalarLeafBuffers(baseLeaf, os, saveFloatAsHalf); + + // Write the number of indices. + Index64 numIndices = Index64(leaf.indices().size()); + os.write(reinterpret_cast(&numIndices), sizeof(Index64)); + + // Write the indices data. + os.write(reinterpret_cast(leaf.indices().data()), numIndices * sizeof(ValueT)); + + // Reserved for future use. + const Index64 auxDataBytes = Index64(0); + os.write(reinterpret_cast(&auxDataBytes), sizeof(Index64)); + } + + std::ostream& os; + const bool saveFloatAsHalf; +}; // struct WritePointIndexBuffersOp + +} // namespace internal + +template +struct PointIndexCodec : public TopologyCodec +{ + using Ptr = std::unique_ptr>; + + ~PointIndexCodec() noexcept = default; + + static inline std::string name() { return GridT::gridType(); } + + void readBuffers(std::istream& is, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics& diagnostics) final + { + GridT& grid = static_cast(*data.grid); + + if (grid.hasMultiPassIO()) { + OPENVDB_THROW(IoError, "Multi-pass IO is not supported in PointIndexCodec"); + } + + io::checkFormatVersion(is); + + bool saveFloatAsHalf = grid.saveFloatAsHalf(); + + auto& tree = grid.tree(); + tree.clearAllAccessors(); + + if (options.clipBBox.isSorted()) { + diagnostics.addWarning(grid.getName(), "bounding box clipping is not supported for PointIndexGrids"); + } + + internal::ReadPointIndexBuffersOp readBuffersOp(is, saveFloatAsHalf, tree.background()); + tools::visitNodesDepthFirst(grid.tree(), readBuffersOp, /*idx=*/0, /*topDown=*/false); + } + + void writeBuffers(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) final + { + const GridT& grid = static_cast(gridBase); + + if (grid.hasMultiPassIO()) { + OPENVDB_THROW(IoError, "Multi-pass IO is not supported in PointIndexCodec"); + } + + internal::WritePointIndexBuffersOp writeBuffersOp(os, grid.saveFloatAsHalf()); + tools::visitNodesDepthFirst(grid.tree(), writeBuffersOp); + } +}; // struct PointIndexCodec + +} // namespace codecs +} // namespace OPENVDB_VERSION_NAME +} // namespace openvdb + +#endif // OPENVDB_IO_CODECS_POINTINDEXCODEC_HAS_BEEN_INCLUDED diff --git a/openvdb/openvdb/io/Codec.cc b/openvdb/openvdb/io/Codec.cc index cfb1489efc..3e437fab17 100644 --- a/openvdb/openvdb/io/Codec.cc +++ b/openvdb/openvdb/io/Codec.cc @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -58,6 +59,7 @@ void initialize() CodecRegistry::registerCodec>(); CodecRegistry::registerCodec>(); + CodecRegistry::registerCodec>(); // register the plugin that converts from scalar to mask/bool NumericGridTypes::foreach(); diff --git a/openvdb/openvdb/unittest/CMakeLists.txt b/openvdb/openvdb/unittest/CMakeLists.txt index 18046cd8e0..446d118027 100644 --- a/openvdb/openvdb/unittest/CMakeLists.txt +++ b/openvdb/openvdb/unittest/CMakeLists.txt @@ -146,6 +146,7 @@ else() TestParticlesToLevelSet.cc TestPointAdvect.cc TestPointAttribute.cc + TestPointCodec.cc TestPointConversion.cc TestPointCount.cc TestPointDataLeaf.cc diff --git a/openvdb/openvdb/unittest/TestPointCodec.cc b/openvdb/openvdb/unittest/TestPointCodec.cc new file mode 100644 index 0000000000..2b8911419e --- /dev/null +++ b/openvdb/openvdb/unittest/TestPointCodec.cc @@ -0,0 +1,147 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include +#include +#include +#include "util.h" // for unittest_util::genPoints + +namespace { + +class PointList { +public: + using PosType = openvdb::Vec3R; + PointList(const std::vector& points) : mPoints(&points) {} + size_t size() const { return mPoints->size(); } + void getPos(size_t n, PosType& xyz) const { xyz = (*mPoints)[n]; } +private: + std::vector const * const mPoints; +}; + +} // namespace + +class TestPointCodec: public ::testing::Test +{ +}; + +TEST_F(TestPointCodec, testPointIndexCodecIO) +{ + using namespace openvdb; + using namespace openvdb::io; + using PointIndexGrid = tools::PointIndexGrid; + + openvdb::initialize(); + CodecRegistry::clear(); + + // Generate points on a unit sphere and build a PointIndexGrid + std::vector points; + unittest_util::genPoints(100, points); + PointList pointList(points); + + const double voxelSize = 0.1; + math::Transform::Ptr transform = math::Transform::createLinearTransform(voxelSize); + + PointIndexGrid::Ptr srcGrid = + tools::createPointIndexGrid(pointList, *transform); + srcGrid->setName("point_index_grid"); + + const std::string rawPath = "testPointIndexCodec_raw.vdb"; + + // Phase 1: write/read without codec + { + io::File f(rawPath); + f.write(GridPtrVec{srcGrid}); + } + + PointIndexGrid::Ptr rawGrid; + { + io::File f(rawPath); + f.open(); + rawGrid = gridPtrCast(f.readGrid("point_index_grid")); + f.close(); + } + ASSERT_TRUE(rawGrid); + + PointIndexGrid::Ptr rawTopo; + { + io::File f(rawPath); + f.open(); + GridBase::Ptr base; + EXPECT_NO_THROW(base = f.readGrid("point_index_grid")); + rawTopo = gridPtrCast(base); + f.close(); + } + ASSERT_TRUE(rawTopo); + EXPECT_EQ(rawTopo->activeVoxelCount(), Index64(97)); + EXPECT_EQ(rawTopo->getName(), std::string("point_index_grid")); + + std::remove(rawPath.c_str()); + + const std::string codecPath = "testPointIndexCodec_codec.vdb"; + + // Phase 2: register codec, write/read with codec + io::internal::initialize(); + ASSERT_TRUE(CodecRegistry::isRegistered(PointIndexGrid::gridType())); + + { + io::File f(codecPath); + f.write(GridPtrVec{srcGrid}); + } + + PointIndexGrid::Ptr codecGrid; + { + io::File f(codecPath); + f.open(); + codecGrid = gridPtrCast(f.readGrid("point_index_grid")); + f.close(); + } + ASSERT_TRUE(codecGrid); + + // Phase 3: full read comparison + EXPECT_TRUE(srcGrid->tree().hasSameTopology(srcGrid->tree())); + EXPECT_TRUE(srcGrid->tree().hasSameTopology(codecGrid->tree())); + { + auto codecAcc = codecGrid->getConstAccessor(); + for (PointIndexGrid::ValueOnCIter it = srcGrid->cbeginValueOn(); it; ++it) { + EXPECT_EQ(*it, codecAcc.getValue(it.getCoord())); + } + } + + // Compare leaf indices arrays + { + auto srcLeafIt = srcGrid->tree().cbeginLeaf(); + auto codecLeafIt = codecGrid->tree().cbeginLeaf(); + for (; srcLeafIt; ++srcLeafIt, ++codecLeafIt) { + ASSERT_TRUE(codecLeafIt); + EXPECT_EQ(srcLeafIt->indices().size(), codecLeafIt->indices().size()); + for (size_t i = 0; i < srcLeafIt->indices().size(); ++i) { + EXPECT_EQ(srcLeafIt->indices()[i], codecLeafIt->indices()[i]); + } + } + EXPECT_TRUE(!codecLeafIt); + } + + // Phase 4: TopologyOnly read + ReadOptions topoOpts; + topoOpts.readMode = ReadMode::TopologyOnly; + + PointIndexGrid::Ptr codecTopo; + { + io::File f(codecPath); + f.open(); + GridBase::Ptr base; + EXPECT_NO_THROW(base = f.readGrid("point_index_grid", topoOpts)); + codecTopo = gridPtrCast(base); + f.close(); + } + ASSERT_TRUE(codecTopo); + EXPECT_EQ(codecTopo->activeVoxelCount(), Index64(0)); + EXPECT_TRUE(codecTopo->tree().leafCount() == 0); + EXPECT_EQ(codecTopo->getName(), std::string("point_index_grid")); + + // Cleanup + CodecRegistry::clear(); + std::remove(codecPath.c_str()); +} From 000fb2473fe385ebfd40de143c4c4fea4fb01a99 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Tue, 17 Feb 2026 11:24:26 -0800 Subject: [PATCH 2/8] Move PointData I/O overloads to a new header Signed-off-by: Dan Bailey --- openvdb/openvdb/CMakeLists.txt | 1 + openvdb/openvdb/points/PointDataGrid.h | 130 +--------------------- openvdb/openvdb/points/PointDataIO.h | 148 +++++++++++++++++++++++++ 3 files changed, 151 insertions(+), 128 deletions(-) create mode 100644 openvdb/openvdb/points/PointDataIO.h diff --git a/openvdb/openvdb/CMakeLists.txt b/openvdb/openvdb/CMakeLists.txt index e9437b86de..97317b1fb7 100644 --- a/openvdb/openvdb/CMakeLists.txt +++ b/openvdb/openvdb/CMakeLists.txt @@ -426,6 +426,7 @@ set(OPENVDB_LIBRARY_POINTS_INCLUDE_FILES points/PointConversion.h points/PointCount.h points/PointDataGrid.h + points/PointDataIO.h points/PointDelete.h points/PointGroup.h points/PointMask.h diff --git a/openvdb/openvdb/points/PointDataGrid.h b/openvdb/openvdb/points/PointDataGrid.h index fcb10bdb2a..946b5764c3 100644 --- a/openvdb/openvdb/points/PointDataGrid.h +++ b/openvdb/openvdb/points/PointDataGrid.h @@ -31,140 +31,14 @@ #include // std::pair, std::make_pair #include +#include // io::readCompressedValues(), io::writeCompressedValues(), io::writeCompressedValuesSize() + class TestPointDataLeaf; namespace openvdb { OPENVDB_USE_VERSION_NAMESPACE namespace OPENVDB_VERSION_NAME { -namespace io -{ - -/// @brief openvdb::io::readCompressedValues specialized on PointDataIndex32 arrays to -/// ignore the value mask, use a larger block size and use 16-bit size instead of 64-bit -template<> -inline void -readCompressedValues( std::istream& is, PointDataIndex32* destBuf, Index destCount, - const util::NodeMask<3>& /*valueMask*/, bool /*fromHalf*/) -{ - using compression::bloscDecompress; - - const bool seek = destBuf == nullptr; - - const size_t destBytes = destCount*sizeof(PointDataIndex32); - const size_t maximumBytes = std::numeric_limits::max(); - if (destBytes >= maximumBytes) { - OPENVDB_THROW(openvdb::IoError, "Cannot read more than " << - maximumBytes << " bytes in voxel values.") - } - - uint16_t bytes16; - - const io::StreamMetadata::Ptr meta = io::getStreamMetadataPtr(is); - - if (seek && meta) { - // buffer size temporarily stored in the StreamMetadata pass - // to avoid having to perform an expensive disk read for 2-bytes - bytes16 = static_cast(meta->pass()); - // seek over size of the compressed buffer - is.seekg(sizeof(uint16_t), std::ios_base::cur); - } - else { - // otherwise read from disk - is.read(reinterpret_cast(&bytes16), sizeof(uint16_t)); - } - - if (bytes16 == std::numeric_limits::max()) { - // read or seek uncompressed data - if (seek) { - is.seekg(destBytes, std::ios_base::cur); - } - else { - is.read(reinterpret_cast(destBuf), destBytes); - } - } - else { - // read or seek uncompressed data - if (seek) { - is.seekg(int(bytes16), std::ios_base::cur); - } - else { - // decompress into the destination buffer - std::unique_ptr bloscBuffer(new char[int(bytes16)]); - is.read(bloscBuffer.get(), bytes16); - std::unique_ptr buffer = bloscDecompress( bloscBuffer.get(), - destBytes, - /*resize=*/false); - std::memcpy(destBuf, buffer.get(), destBytes); - } - } -} - -/// @brief openvdb::io::writeCompressedValues specialized on PointDataIndex32 arrays to -/// ignore the value mask, use a larger block size and use 16-bit size instead of 64-bit -template<> -inline void -writeCompressedValues( std::ostream& os, const PointDataIndex32* srcBuf, Index srcCount, - const util::NodeMask<3>& /*valueMask*/, - const util::NodeMask<3>& /*childMask*/, bool /*toHalf*/) -{ - using compression::bloscCompress; - - const size_t srcBytes = srcCount*sizeof(PointDataIndex32); - const size_t maximumBytes = std::numeric_limits::max(); - if (srcBytes >= maximumBytes) { - OPENVDB_THROW(openvdb::IoError, "Cannot write more than " << - maximumBytes << " bytes in voxel values.") - } - - const char* charBuffer = reinterpret_cast(srcBuf); - - size_t compressedBytes; - std::unique_ptr buffer = bloscCompress( charBuffer, srcBytes, - compressedBytes, /*resize=*/false); - - if (compressedBytes > 0) { - auto bytes16 = static_cast(compressedBytes); // clamp to 16-bit unsigned integer - os.write(reinterpret_cast(&bytes16), sizeof(uint16_t)); - os.write(reinterpret_cast(buffer.get()), compressedBytes); - } - else { - auto bytes16 = static_cast(maximumBytes); // max value indicates uncompressed - os.write(reinterpret_cast(&bytes16), sizeof(uint16_t)); - os.write(reinterpret_cast(srcBuf), srcBytes); - } -} - -template -inline void -writeCompressedValuesSize(std::ostream& os, const T* srcBuf, Index srcCount) -{ - using compression::bloscCompressedSize; - - const size_t srcBytes = srcCount*sizeof(T); - const size_t maximumBytes = std::numeric_limits::max(); - if (srcBytes >= maximumBytes) { - OPENVDB_THROW(openvdb::IoError, "Cannot write more than " << - maximumBytes << " bytes in voxel values.") - } - - const char* charBuffer = reinterpret_cast(srcBuf); - - // calculate voxel buffer size after compression - size_t compressedBytes = bloscCompressedSize(charBuffer, srcBytes); - - if (compressedBytes > 0) { - auto bytes16 = static_cast(compressedBytes); // clamp to 16-bit unsigned integer - os.write(reinterpret_cast(&bytes16), sizeof(uint16_t)); - } - else { - auto bytes16 = static_cast(maximumBytes); // max value indicates uncompressed - os.write(reinterpret_cast(&bytes16), sizeof(uint16_t)); - } -} - -} // namespace io - // forward declaration namespace tree { diff --git a/openvdb/openvdb/points/PointDataIO.h b/openvdb/openvdb/points/PointDataIO.h new file mode 100644 index 0000000000..fddbaef6c4 --- /dev/null +++ b/openvdb/openvdb/points/PointDataIO.h @@ -0,0 +1,148 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 + +#ifndef OPENVDB_POINTS_POINT_DATA_IO_HAS_BEEN_INCLUDED +#define OPENVDB_POINTS_POINT_DATA_IO_HAS_BEEN_INCLUDED + + +namespace openvdb { +OPENVDB_USE_VERSION_NAMESPACE +namespace OPENVDB_VERSION_NAME { + + +//////////////////////////////////////// + + +namespace io +{ + +/// @brief openvdb::io::readCompressedValues specialized on PointDataIndex32 arrays to +/// ignore the value mask, use a larger block size and use 16-bit size instead of 64-bit +template<> +inline void +readCompressedValues( std::istream& is, PointDataIndex32* destBuf, Index destCount, + const util::NodeMask<3>& /*valueMask*/, bool /*fromHalf*/) +{ + using compression::bloscDecompress; + + const bool seek = destBuf == nullptr; + + const size_t destBytes = destCount*sizeof(PointDataIndex32); + const size_t maximumBytes = std::numeric_limits::max(); + if (destBytes >= maximumBytes) { + OPENVDB_THROW(openvdb::IoError, "Cannot read more than " << + maximumBytes << " bytes in voxel values.") + } + + uint16_t bytes16; + + const io::StreamMetadata::Ptr meta = io::getStreamMetadataPtr(is); + + if (seek && meta) { + // buffer size temporarily stored in the StreamMetadata pass + // to avoid having to perform an expensive disk read for 2-bytes + bytes16 = static_cast(meta->pass()); + // seek over size of the compressed buffer + is.seekg(sizeof(uint16_t), std::ios_base::cur); + } + else { + // otherwise read from disk + is.read(reinterpret_cast(&bytes16), sizeof(uint16_t)); + } + + if (bytes16 == std::numeric_limits::max()) { + // read or seek uncompressed data + if (seek) { + is.seekg(destBytes, std::ios_base::cur); + } + else { + is.read(reinterpret_cast(destBuf), destBytes); + } + } + else { + // read or seek uncompressed data + if (seek) { + is.seekg(int(bytes16), std::ios_base::cur); + } + else { + // decompress into the destination buffer + std::unique_ptr bloscBuffer(new char[int(bytes16)]); + is.read(bloscBuffer.get(), bytes16); + std::unique_ptr buffer = bloscDecompress( bloscBuffer.get(), + destBytes, + /*resize=*/false); + std::memcpy(destBuf, buffer.get(), destBytes); + } + } +} + +/// @brief openvdb::io::writeCompressedValues specialized on PointDataIndex32 arrays to +/// ignore the value mask, use a larger block size and use 16-bit size instead of 64-bit +template<> +inline void +writeCompressedValues( std::ostream& os, const PointDataIndex32* srcBuf, Index srcCount, + const util::NodeMask<3>& /*valueMask*/, + const util::NodeMask<3>& /*childMask*/, bool /*toHalf*/) +{ + using compression::bloscCompress; + + const size_t srcBytes = srcCount*sizeof(PointDataIndex32); + const size_t maximumBytes = std::numeric_limits::max(); + if (srcBytes >= maximumBytes) { + OPENVDB_THROW(openvdb::IoError, "Cannot write more than " << + maximumBytes << " bytes in voxel values.") + } + + const char* charBuffer = reinterpret_cast(srcBuf); + + size_t compressedBytes; + std::unique_ptr buffer = bloscCompress( charBuffer, srcBytes, + compressedBytes, /*resize=*/false); + + if (compressedBytes > 0) { + auto bytes16 = static_cast(compressedBytes); // clamp to 16-bit unsigned integer + os.write(reinterpret_cast(&bytes16), sizeof(uint16_t)); + os.write(reinterpret_cast(buffer.get()), compressedBytes); + } + else { + auto bytes16 = static_cast(maximumBytes); // max value indicates uncompressed + os.write(reinterpret_cast(&bytes16), sizeof(uint16_t)); + os.write(reinterpret_cast(srcBuf), srcBytes); + } +} + +template +inline void +writeCompressedValuesSize(std::ostream& os, const T* srcBuf, Index srcCount) +{ + using compression::bloscCompressedSize; + + const size_t srcBytes = srcCount*sizeof(T); + const size_t maximumBytes = std::numeric_limits::max(); + if (srcBytes >= maximumBytes) { + OPENVDB_THROW(openvdb::IoError, "Cannot write more than " << + maximumBytes << " bytes in voxel values.") + } + + const char* charBuffer = reinterpret_cast(srcBuf); + + // calculate voxel buffer size after compression + size_t compressedBytes = bloscCompressedSize(charBuffer, srcBytes); + + if (compressedBytes > 0) { + auto bytes16 = static_cast(compressedBytes); // clamp to 16-bit unsigned integer + os.write(reinterpret_cast(&bytes16), sizeof(uint16_t)); + } + else { + auto bytes16 = static_cast(maximumBytes); // max value indicates uncompressed + os.write(reinterpret_cast(&bytes16), sizeof(uint16_t)); + } +} + +} // namespace io + + +} // namespace OPENVDB_VERSION_NAME +} // namespace openvdb + +#endif // OPENVDB_POINTS_POINT_DATA_IO_HAS_BEEN_INCLUDED From 6528cb82eb0691514b60f74ac4127026b53d4ee6 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Wed, 18 Mar 2026 14:18:25 -0700 Subject: [PATCH 3/8] Remove support for multi-pass I/O except PointDataLeafNode Signed-off-by: Dan Bailey --- openvdb/openvdb/Grid.h | 30 +++- openvdb/openvdb/io/io.h | 4 +- openvdb/openvdb/unittest/TestFile.cc | 240 --------------------------- 3 files changed, 27 insertions(+), 247 deletions(-) diff --git a/openvdb/openvdb/Grid.h b/openvdb/openvdb/Grid.h index b364e9efc0..b2f1dd8ecd 100644 --- a/openvdb/openvdb/Grid.h +++ b/openvdb/openvdb/Grid.h @@ -1181,14 +1181,34 @@ struct TreeAdapter > //////////////////////////////////////// +namespace points { + +template class PointDataLeafNode; + +/// @brief Type trait that evaluates to true only for @c PointDataLeafNode instantiations. +template +struct IsPointDataLeafNode : std::false_type {}; + +template +struct IsPointDataLeafNode> : std::true_type {}; + +} // namespace points + + /// @brief Metafunction that specifies whether a given leaf node, tree, or grid type -/// requires multiple passes to read and write voxel data -/// @details Multi-pass I/O allows one to optimize the data layout of leaf nodes -/// for certain access patterns during delayed loading. -/// @sa io::MultiPass +/// requires multiple passes to read and write voxel data. +/// @details Multi-pass I/O allows leaf nodes to optimize their serialization layout +/// for delayed-load access patterns. Only @c PointDataLeafNode supports multi-pass I/O. +/// Defining a custom leaf node that inherits @c io::MultiPass is no longer permitted. +/// @sa points::IsPointDataLeafNode template struct HasMultiPassIO { - static const bool value = std::is_base_of::value; + static_assert( + !std::is_base_of::value + || points::IsPointDataLeafNode::value, + "Only PointDataLeafNode may inherit from io::MultiPass; " + "use points::IsPointDataLeafNode to test for multi-pass I/O support."); + static const bool value = points::IsPointDataLeafNode::value; }; // Partial specialization for Tree types diff --git a/openvdb/openvdb/io/io.h b/openvdb/openvdb/io/io.h index f2ac342ff7..96d2408eb3 100644 --- a/openvdb/openvdb/io/io.h +++ b/openvdb/openvdb/io/io.h @@ -116,8 +116,8 @@ std::ostream& operator<<(std::ostream&, const StreamMetadata::AuxDataMap&); //////////////////////////////////////// -/// @brief Leaf nodes that require multi-pass I/O must inherit from this struct. -/// @sa Grid::hasMultiPassIO() +/// @brief Tag for multi-pass I/O. Only points::PointDataLeafNode may inherit from this. +/// @sa Grid::hasMultiPassIO(), points::IsPointDataLeafNode struct MultiPass {}; diff --git a/openvdb/openvdb/unittest/TestFile.cc b/openvdb/openvdb/unittest/TestFile.cc index fad51e9567..130947fdf1 100644 --- a/openvdb/openvdb/unittest/TestFile.cc +++ b/openvdb/openvdb/unittest/TestFile.cc @@ -1522,246 +1522,6 @@ TEST_F(TestFile, testReadClippedGrid) //////////////////////////////////////// -namespace { - -template struct MultiPassLeafNode; // forward declaration - -// Dummy value type -using MultiPassValue = openvdb::PointIndex; - -// Tree configured to match the default OpenVDB configuration -using MultiPassTree = openvdb::tree::Tree< - openvdb::tree::RootNode< - openvdb::tree::InternalNode< - openvdb::tree::InternalNode< - MultiPassLeafNode, 4>, 5>>>; - -using MultiPassGrid = openvdb::Grid; - - -template -struct MultiPassLeafNode: public openvdb::tree::LeafNode, openvdb::io::MultiPass -{ - // The following had to be copied from the LeafNode class - // to make the derived class compatible with the tree structure. - - using LeafNodeType = MultiPassLeafNode; - using Ptr = openvdb::SharedPtr; - using BaseLeaf = openvdb::tree::LeafNode; - using NodeMaskType = openvdb::util::NodeMask; - using ValueType = T; - using ValueOnCIter = typename BaseLeaf::template ValueIter; - using ChildOnIter = typename BaseLeaf::template ChildIter; - using ChildOnCIter = typename BaseLeaf::template ChildIter< - typename NodeMaskType::OnIterator, const MultiPassLeafNode, typename BaseLeaf::ChildOn>; - - MultiPassLeafNode(const openvdb::Coord& coords, const T& value, bool active = false) - : BaseLeaf(coords, value, active) {} - MultiPassLeafNode(openvdb::PartialCreate, const openvdb::Coord& coords, const T& value, - bool active = false): BaseLeaf(openvdb::PartialCreate(), coords, value, active) {} - MultiPassLeafNode(const MultiPassLeafNode& rhs): BaseLeaf(rhs) {} - - ValueOnCIter cbeginValueOn() const { return ValueOnCIter(this->getValueMask().beginOn(),this); } - ChildOnCIter cbeginChildOn() const { return ChildOnCIter(this->getValueMask().endOn(), this); } - ChildOnIter beginChildOn() { return ChildOnIter(this->getValueMask().endOn(), this); } - - // Methods in use for reading and writing multiple buffers - - void readBuffers(std::istream& is, const openvdb::CoordBBox&, bool fromHalf = false) - { - this->readBuffers(is, fromHalf); - } - - void readBuffers(std::istream& is, bool /*fromHalf*/ = false) - { - const openvdb::io::StreamMetadata::Ptr meta = openvdb::io::getStreamMetadataPtr(is); - if (!meta) { - OPENVDB_THROW(openvdb::IoError, - "Cannot write out a MultiBufferLeaf without StreamMetadata."); - } - - // clamp pass to 16-bit integer - const uint32_t pass(static_cast(meta->pass())); - - // Read in the stored pass number. - uint32_t readPass; - is.read(reinterpret_cast(&readPass), sizeof(uint32_t)); - EXPECT_EQ(pass, readPass); - // Record the pass number. - mReadPasses.push_back(readPass); - - if (pass == 0) { - // Read in the node's origin. - openvdb::Coord origin; - is.read(reinterpret_cast(&origin), sizeof(openvdb::Coord)); - EXPECT_EQ(origin, this->origin()); - } - } - - void writeBuffers(std::ostream& os, bool /*toHalf*/ = false) const - { - const openvdb::io::StreamMetadata::Ptr meta = openvdb::io::getStreamMetadataPtr(os); - if (!meta) { - OPENVDB_THROW(openvdb::IoError, - "Cannot read in a MultiBufferLeaf without StreamMetadata."); - } - - // clamp pass to 16-bit integer - const uint32_t pass(static_cast(meta->pass())); - - // Leaf traversal analysis deduces the number of passes to perform for this leaf - // then updates the leaf traversal value to ensure all passes will be written. - if (meta->countingPasses()) { - if (mNumPasses > pass) meta->setPass(mNumPasses); - return; - } - - // Record the pass number. - EXPECT_TRUE(mWritePassesPtr); - const_cast&>(*mWritePassesPtr).push_back(pass); - - // Write out the pass number. - os.write(reinterpret_cast(&pass), sizeof(uint32_t)); - if (pass == 0) { - // Write out the node's origin and the pass number. - const auto origin = this->origin(); - os.write(reinterpret_cast(&origin), sizeof(openvdb::Coord)); - } - } - - - uint32_t mNumPasses = 0; - // Pointer to external vector in which to record passes as they are written - std::vector* mWritePassesPtr = nullptr; - // Vector in which to record passes as they are read - // (this needs to be internal, because leaf nodes are constructed as a grid is read) - std::vector mReadPasses; -}; // struct MultiPassLeafNode - -} // anonymous namespace - - -TEST_F(TestFile, testMultiPassIO) -{ - using namespace openvdb; - - openvdb::initialize(); - MultiPassGrid::registerGrid(); - - // Create a multi-buffer grid. - const MultiPassGrid::Ptr grid = openvdb::createGrid(); - grid->setName("test"); - grid->setTransform(math::Transform::createLinearTransform(1.0)); - MultiPassGrid::TreeType& tree = grid->tree(); - tree.setValue(Coord(0, 0, 0), 5); - tree.setValue(Coord(0, 10, 0), 5); - EXPECT_EQ(2, int(tree.leafCount())); - - const GridPtrVec grids{grid}; - - // Vector in which to record pass numbers (to ensure blocked ordering) - std::vector writePasses; - { - // Specify the required number of I/O passes for each leaf node. - MultiPassGrid::TreeType::LeafIter leafIter = tree.beginLeaf(); - leafIter->mNumPasses = 3; - leafIter->mWritePassesPtr = &writePasses; - ++leafIter; - leafIter->mNumPasses = 2; - leafIter->mWritePassesPtr = &writePasses; - } - - const char* filename = "testMultiPassIO.vdb"; - SharedPtr scopedFile(filename, ::remove); - { - // Verify that passes are written to a file in the correct order. - io::File(filename).write(grids); - EXPECT_EQ(6, int(writePasses.size())); - EXPECT_EQ(0, writePasses[0]); // leaf 0 - EXPECT_EQ(0, writePasses[1]); // leaf 1 - EXPECT_EQ(1, writePasses[2]); // leaf 0 - EXPECT_EQ(1, writePasses[3]); // leaf 1 - EXPECT_EQ(2, writePasses[4]); // leaf 0 - EXPECT_EQ(2, writePasses[5]); // leaf 1 - } - { - // Verify that passes are read in the correct order. - io::File file(filename); - file.open(); - const auto newGrid = GridBase::grid(file.readGrid("test")); - - auto leafIter = newGrid->tree().beginLeaf(); - EXPECT_EQ(3, int(leafIter->mReadPasses.size())); - EXPECT_EQ(0, leafIter->mReadPasses[0]); - EXPECT_EQ(1, leafIter->mReadPasses[1]); - EXPECT_EQ(2, leafIter->mReadPasses[2]); - ++leafIter; - EXPECT_EQ(3, int(leafIter->mReadPasses.size())); - EXPECT_EQ(0, leafIter->mReadPasses[0]); - EXPECT_EQ(1, leafIter->mReadPasses[1]); - EXPECT_EQ(2, leafIter->mReadPasses[2]); - } - { - // Verify that when using multi-pass and bbox clipping that each leaf node - // is still being read before being clipped - io::File file(filename); - file.open(); - const auto newGrid = GridBase::grid( - file.readGrid("test", BBoxd(Vec3d(0), Vec3d(1)))); - EXPECT_EQ(Index64(1), newGrid->tree().leafCount()); - - auto leafIter = newGrid->tree().beginLeaf(); - EXPECT_EQ(3, int(leafIter->mReadPasses.size())); - EXPECT_EQ(0, leafIter->mReadPasses[0]); - EXPECT_EQ(1, leafIter->mReadPasses[1]); - EXPECT_EQ(2, leafIter->mReadPasses[2]); - ++leafIter; - EXPECT_TRUE(!leafIter); // second leaf node has now been clipped - } - - // Clear the pass data. - writePasses.clear(); - - { - // Verify that passes are written to and read from a non-seekable stream - // in the correct order. - std::ostringstream ostr(std::ios_base::binary); - io::Stream(ostr).write(grids); - - EXPECT_EQ(6, int(writePasses.size())); - EXPECT_EQ(0, writePasses[0]); // leaf 0 - EXPECT_EQ(0, writePasses[1]); // leaf 1 - EXPECT_EQ(1, writePasses[2]); // leaf 0 - EXPECT_EQ(1, writePasses[3]); // leaf 1 - EXPECT_EQ(2, writePasses[4]); // leaf 0 - EXPECT_EQ(2, writePasses[5]); // leaf 1 - - std::istringstream is(ostr.str(), std::ios_base::binary); - io::Stream strm(is); - const auto streamedGrids = strm.getGrids(); - EXPECT_EQ(1, int(streamedGrids->size())); - - const auto newGrid = gridPtrCast(*streamedGrids->begin()); - EXPECT_TRUE(bool(newGrid)); - auto leafIter = newGrid->tree().beginLeaf(); - EXPECT_EQ(3, int(leafIter->mReadPasses.size())); - EXPECT_EQ(0, leafIter->mReadPasses[0]); - EXPECT_EQ(1, leafIter->mReadPasses[1]); - EXPECT_EQ(2, leafIter->mReadPasses[2]); - ++leafIter; - EXPECT_EQ(3, int(leafIter->mReadPasses.size())); - EXPECT_EQ(0, leafIter->mReadPasses[0]); - EXPECT_EQ(1, leafIter->mReadPasses[1]); - EXPECT_EQ(2, leafIter->mReadPasses[2]); - } -} - - -//////////////////////////////////////// - - TEST_F(TestFile, testHasGrid) { using namespace openvdb; From 0b95d2e9ca190858b4b7fb23b963860f138f8eeb Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Wed, 18 Mar 2026 14:16:25 -0700 Subject: [PATCH 4/8] Remove AttributeArray out-of-core atomic Signed-off-by: Dan Bailey --- openvdb/openvdb/points/AttributeArray.cc | 2 -- openvdb/openvdb/points/AttributeArray.h | 8 +++----- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/openvdb/openvdb/points/AttributeArray.cc b/openvdb/openvdb/points/AttributeArray.cc index 7d83051b52..19e6d14436 100644 --- a/openvdb/openvdb/points/AttributeArray.cc +++ b/openvdb/openvdb/points/AttributeArray.cc @@ -61,7 +61,6 @@ AttributeArray::AttributeArray(const AttributeArray& rhs, const tbb::spin_mutex: : mIsUniform(rhs.mIsUniform) , mFlags(rhs.mFlags) , mUsePagedRead(rhs.mUsePagedRead) - , mOutOfCore(rhs.mOutOfCore.load()) , mPageHandle() { if (mFlags & PARTIALREAD) mCompressedBytes = rhs.mCompressedBytes; @@ -78,7 +77,6 @@ AttributeArray::operator=(const AttributeArray& rhs) mIsUniform = rhs.mIsUniform; mFlags = rhs.mFlags; mUsePagedRead = rhs.mUsePagedRead; - mOutOfCore.store(rhs.mOutOfCore); if (mFlags & PARTIALREAD) mCompressedBytes = rhs.mCompressedBytes; else if (rhs.mPageHandle) mPageHandle = rhs.mPageHandle->copy(); else mPageHandle.reset(); diff --git a/openvdb/openvdb/points/AttributeArray.h b/openvdb/openvdb/points/AttributeArray.h index bbfba7b156..3ad2d3a48d 100644 --- a/openvdb/openvdb/points/AttributeArray.h +++ b/openvdb/openvdb/points/AttributeArray.h @@ -363,10 +363,12 @@ class OPENVDB_API AttributeArray mutable tbb::spin_mutex mMutex; uint8_t mFlags = 0; uint8_t mUsePagedRead = 0; +#if OPENVDB_ABI_VERSION_NUMBER < 14 std::atomic mOutOfCore{0}; // interpreted as bool +#endif /// used for out-of-core, paged reading union { - compression::PageHandle::Ptr mPageHandle; + std::unique_ptr mPageHandle; size_t mCompressedBytes; }; }; // class AttributeArray @@ -1535,8 +1537,6 @@ TypedAttributeArray::readBuffers(std::istream& is) OPENVDB_THROW(IoError, "Cannot read paged AttributeArray buffers."); } - tbb::spin_mutex::scoped_lock lock(mMutex); - this->deallocate(); uint8_t bloscCompressed(0); @@ -1586,8 +1586,6 @@ TypedAttributeArray::readPagedBuffers(compression::PagedInpu OPENVDB_ASSERT(mPageHandle); - tbb::spin_mutex::scoped_lock lock(mMutex); - this->deallocate(); is.read(mPageHandle, std::streamsize(mPageHandle->size()), false); From a9a6cff2ab5a8c248c0319ee2d2545c20ac28157 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Fri, 27 Mar 2026 14:36:30 -0700 Subject: [PATCH 5/8] Rename MultiPass tag class to PointDataGridMultiPass Signed-off-by: Dan Bailey --- openvdb/openvdb/Grid.h | 6 +++--- openvdb/openvdb/io/io.h | 2 +- openvdb/openvdb/points/PointDataGrid.h | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/openvdb/openvdb/Grid.h b/openvdb/openvdb/Grid.h index b2f1dd8ecd..41f6e58501 100644 --- a/openvdb/openvdb/Grid.h +++ b/openvdb/openvdb/Grid.h @@ -1199,14 +1199,14 @@ struct IsPointDataLeafNode> : std::true_type {}; /// requires multiple passes to read and write voxel data. /// @details Multi-pass I/O allows leaf nodes to optimize their serialization layout /// for delayed-load access patterns. Only @c PointDataLeafNode supports multi-pass I/O. -/// Defining a custom leaf node that inherits @c io::MultiPass is no longer permitted. +/// Defining a custom leaf node that inherits @c io::PointDataGridMultiPass is no longer permitted. /// @sa points::IsPointDataLeafNode template struct HasMultiPassIO { static_assert( - !std::is_base_of::value + !std::is_base_of::value || points::IsPointDataLeafNode::value, - "Only PointDataLeafNode may inherit from io::MultiPass; " + "Only PointDataLeafNode may inherit from io::PointDataGridMultiPass; " "use points::IsPointDataLeafNode to test for multi-pass I/O support."); static const bool value = points::IsPointDataLeafNode::value; }; diff --git a/openvdb/openvdb/io/io.h b/openvdb/openvdb/io/io.h index 96d2408eb3..13a9134012 100644 --- a/openvdb/openvdb/io/io.h +++ b/openvdb/openvdb/io/io.h @@ -118,7 +118,7 @@ std::ostream& operator<<(std::ostream&, const StreamMetadata::AuxDataMap&); /// @brief Tag for multi-pass I/O. Only points::PointDataLeafNode may inherit from this. /// @sa Grid::hasMultiPassIO(), points::IsPointDataLeafNode -struct MultiPass {}; +struct PointDataGridMultiPass {}; //////////////////////////////////////// diff --git a/openvdb/openvdb/points/PointDataGrid.h b/openvdb/openvdb/points/PointDataGrid.h index 946b5764c3..f186717799 100644 --- a/openvdb/openvdb/points/PointDataGrid.h +++ b/openvdb/openvdb/points/PointDataGrid.h @@ -107,7 +107,7 @@ prefetch(PointDataTreeT&, bool /*position*/ = true, bool /*otherAttributes*/ = t template -class PointDataLeafNode : public tree::LeafNode, io::MultiPass { +class PointDataLeafNode : public tree::LeafNode, io::PointDataGridMultiPass { public: using LeafNodeType = PointDataLeafNode; From 646cdaa609ae204d4095d1823a901a8c32d182b8 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Mon, 20 Apr 2026 11:25:29 -0700 Subject: [PATCH 6/8] Add PointDataCodec Signed-off-by: Dan Bailey --- openvdb/openvdb/CMakeLists.txt | 1 + openvdb/openvdb/codecs/PointDataCodec.h | 449 ++++++++++++++++++++ openvdb/openvdb/io/Codec.cc | 2 + openvdb/openvdb/points/AttributeArray.h | 40 ++ openvdb/openvdb/points/StreamCompression.cc | 32 ++ openvdb/openvdb/points/StreamCompression.h | 8 + openvdb/openvdb/unittest/TestPointCodec.cc | 400 +++++++++++++++++ 7 files changed, 932 insertions(+) create mode 100644 openvdb/openvdb/codecs/PointDataCodec.h diff --git a/openvdb/openvdb/CMakeLists.txt b/openvdb/openvdb/CMakeLists.txt index 97317b1fb7..19172ca380 100644 --- a/openvdb/openvdb/CMakeLists.txt +++ b/openvdb/openvdb/CMakeLists.txt @@ -367,6 +367,7 @@ set(OPENVDB_LIBRARY_INCLUDE_FILES set(OPENVDB_LIBRARY_CODECS_INCLUDE_FILES codecs/BoolCodec.h + codecs/PointDataCodec.h codecs/PointIndexCodec.h codecs/ScalarCodec.h codecs/TopologyCodec.h diff --git a/openvdb/openvdb/codecs/PointDataCodec.h b/openvdb/openvdb/codecs/PointDataCodec.h new file mode 100644 index 0000000000..e5cc550fe4 --- /dev/null +++ b/openvdb/openvdb/codecs/PointDataCodec.h @@ -0,0 +1,449 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 + +#ifndef OPENVDB_IO_CODECS_POINTDATACODEC_HAS_BEEN_INCLUDED +#define OPENVDB_IO_CODECS_POINTDATACODEC_HAS_BEEN_INCLUDED + +#include + +#include + +#include +#include +#include + +#include "ScalarLeafCodec.h" +#include "TopologyCodec.h" + +#include +#include + +namespace openvdb { +OPENVDB_USE_VERSION_NAMESPACE +namespace OPENVDB_VERSION_NAME { +namespace codecs { +namespace internal { + +/// Look up an existing paged stream or create a new one for the given attribute index +template +inline typename PagedStreamPtrT::element_type* getOrCreatePagedStream( + std::map& pagedStreams, Index attributeIndex) +{ + auto it = pagedStreams.find(attributeIndex); + if (it != pagedStreams.end()) return it->second.get(); + using PagedStreamT = typename PagedStreamPtrT::element_type; + pagedStreams[attributeIndex] = std::make_shared(); + return pagedStreams[attributeIndex].get(); +} + +//////////////////////////////////////// +// Read-side functions + +template +inline void readPointDataVoxelSizes(const std::vector& leaves, + std::istream& is, std::map& voxelBufferSizes) +{ + for (auto* leaf : leaves) { + uint16_t voxelBufferSize; + is.read(reinterpret_cast(&voxelBufferSize), sizeof(uint16_t)); + voxelBufferSizes[leaf->origin()] = voxelBufferSize; + } +} + +template +inline void readPointDataDescriptors(const std::vector& leaves, + std::istream& is) +{ + points::AttributeSet::Descriptor::Ptr sharedDescriptor; + for (auto* leaf : leaves) { + points::AttributeSet::UniquePtr attrSet = leaf->stealAttributeSet(); + if (sharedDescriptor) { + // Reuse shared descriptor from first leaf + attrSet->resetDescriptor(sharedDescriptor, /*allowMismatchingDescriptors=*/true); + } + else { + uint8_t header; + is.read(reinterpret_cast(&header), sizeof(uint8_t)); + attrSet->readDescriptor(is); + if (header & uint8_t(1)) { + // Store descriptor for subsequent leaves + sharedDescriptor = attrSet->descriptorPtr(); + } + // a forwards-compatibility mechanism for future use, + // if a 0x2 bit is set, read and skip over a specific number of bytes + if (header & uint8_t(2)) { + uint64_t bytesToSkip; + is.read(reinterpret_cast(&bytesToSkip), sizeof(uint64_t)); + if (bytesToSkip > uint64_t(0)) { + std::vector tempData(bytesToSkip); + is.read(reinterpret_cast(&tempData[0]), bytesToSkip); + } + } + // this reader is only able to read headers with 0x1 and 0x2 bits set + if (header > uint8_t(3)) { + OPENVDB_THROW(IoError, "Unrecognised header flags in PointDataLeafNode"); + } + } + attrSet->readMetadata(is); + leaf->replaceAttributeSet(attrSet.release(), /*allowMismatchingDescriptors=*/true); + } +} + +template +inline void readPointDataAttributeSizes(const std::vector& leaves, + std::istream& is, Index attributeIndex, + std::map& pagedStreams) +{ + for (auto* leaf : leaves) { + points::AttributeArray* array = attributeIndex < leaf->attributeSet().size() ? + &leaf->attributeArray(attributeIndex) : nullptr; + if (array) { + auto* pagedStream = getOrCreatePagedStream(pagedStreams, attributeIndex); + pagedStream->setInputStream(is); + pagedStream->setSizeOnly(true); + array->readPagedBuffers(*pagedStream); + } + } +} + +template +inline void readPointDataVoxelData(const std::vector& leaves, + std::istream& is, bool saveFloatAsHalf, + const typename LeafT::ValueType& background, + const std::map& voxelBufferSizes) +{ + (void) voxelBufferSizes; + using BaseLeaf = typename LeafT::BaseLeaf; + for (auto* leaf : leaves) { + OPENVDB_ASSERT(voxelBufferSizes.find(leaf->origin()) != voxelBufferSizes.end()); + BaseLeaf& baseLeaf = static_cast(*leaf); + readScalarLeafBuffers(baseLeaf, is, saveFloatAsHalf, background); + } +} + +template +inline void readPointDataAttributeData(const std::vector& leaves, + std::istream& is, Index attributeIndex, + std::map& pagedStreams) +{ + for (auto* leaf : leaves) { + points::AttributeArray* array = attributeIndex < leaf->attributeSet().size() ? + &leaf->attributeArray(attributeIndex) : nullptr; + if (array) { + auto* pagedStream = getOrCreatePagedStream(pagedStreams, attributeIndex); + pagedStream->setInputStream(is); + pagedStream->setSizeOnly(false); + array->readPagedBuffers(*pagedStream); + } + } +} + +template +inline void skipPointDataAttributeSizes(const std::vector& leaves, + std::istream& is, Index attributeIndex, + std::map& pagedStreams) +{ + for (auto* leaf : leaves) { + points::AttributeArray* array = attributeIndex < leaf->attributeSet().size() ? + &leaf->attributeArray(attributeIndex) : nullptr; + if (array) { + auto* pagedStream = getOrCreatePagedStream(pagedStreams, attributeIndex); + pagedStream->setInputStream(is); + pagedStream->setSizeOnly(true); + array->skipPagedBuffers(*pagedStream); + } + } +} + +template +inline void skipPointDataAttributeData(const std::vector& leaves, + std::istream& is, Index attributeIndex, + std::map& pagedStreams) +{ + for (auto* leaf : leaves) { + points::AttributeArray* array = attributeIndex < leaf->attributeSet().size() ? + &leaf->attributeArray(attributeIndex) : nullptr; + if (array) { + auto* pagedStream = getOrCreatePagedStream(pagedStreams, attributeIndex); + pagedStream->setInputStream(is); + pagedStream->setSizeOnly(false); + array->skipPagedBuffers(*pagedStream); + } + } +} + +//////////////////////////////////////// +// Write-side functions + +template +inline Index countPointDataPasses(const std::vector& leaves) +{ + Index maxRequiredPasses = 0; + for (const auto* leaf : leaves) { + const Index requiredPasses = leaf->buffers(); + if (requiredPasses > maxRequiredPasses) { + maxRequiredPasses = requiredPasses; + } + } + return maxRequiredPasses; +} + +template +inline void writePointDataVoxelSizes(const std::vector& leaves, + std::ostream& os, bool& matching, + points::AttributeSet::Descriptor::Ptr& sharedDescriptor) +{ + bool descriptorChecked = false; + matching = true; + for (const auto* leaf : leaves) { + io::writeCompressedValuesSize(os, leaf->buffer().data(), LeafT::SIZE); + + // Track descriptor matching + const auto& descriptor = leaf->attributeSet().descriptorPtr(); + if (!descriptorChecked) { + // First leaf - store descriptor + descriptorChecked = true; + sharedDescriptor = descriptor; + } + else if (matching && *sharedDescriptor != *descriptor) { + matching = false; + } + } +} + +template +inline void writePointDataDescriptors(const std::vector& leaves, + std::ostream& os, bool matching, + const points::AttributeSet::Descriptor::Ptr&) +{ + bool firstWrite = true; + for (const auto* leaf : leaves) { + const points::AttributeSet& attributeSet = leaf->attributeSet(); + if (matching) { + // Shared descriptor - only write on first leaf + if (firstWrite) { + firstWrite = false; + uint8_t header(1); + os.write(reinterpret_cast(&header), sizeof(uint8_t)); + attributeSet.writeDescriptor(os, /*transient=*/false); + } + } + else { + // Non-shared descriptor - write on every leaf + uint8_t header(0); + os.write(reinterpret_cast(&header), sizeof(uint8_t)); + attributeSet.writeDescriptor(os, /*transient=*/false); + } + attributeSet.writeMetadata(os, /*transient=*/false, /*paged=*/true); + } +} + +template +inline void writePointDataAttributeSizes(const std::vector& leaves, + std::ostream& os, Index attributeIndex) +{ + std::map pagedStreams; + for (const auto* leaf : leaves) { + const points::AttributeArray* array = attributeIndex < leaf->attributeSet().size() ? + leaf->attributeSet().getConst(attributeIndex) : nullptr; + if (array) { + auto* pagedStream = getOrCreatePagedStream(pagedStreams, attributeIndex); + pagedStream->setOutputStream(os); + pagedStream->setSizeOnly(true); + array->writePagedBuffers(*pagedStream, /*outputTransient*/false); + } + } + // Flush paged streams to write any remaining buffered page headers + for (auto& pair : pagedStreams) { + pair.second->flush(); + } +} + +template +inline void writePointDataVoxelData(const std::vector& leaves, + std::ostream& os, bool saveFloatAsHalf) +{ + using BaseLeaf = typename LeafT::BaseLeaf; + for (const auto* leaf : leaves) { + const BaseLeaf& baseLeaf = static_cast(*leaf); + writeScalarLeafBuffers(baseLeaf, os, saveFloatAsHalf); + } +} + +template +inline void writePointDataAttributeData(const std::vector& leaves, + std::ostream& os, Index attributeIndex) +{ + std::map pagedStreams; + for (const auto* leaf : leaves) { + const points::AttributeArray* array = attributeIndex < leaf->attributeSet().size() ? + leaf->attributeSet().getConst(attributeIndex) : nullptr; + if (array) { + auto* pagedStream = getOrCreatePagedStream(pagedStreams, attributeIndex); + pagedStream->setOutputStream(os); + pagedStream->setSizeOnly(false); + array->writePagedBuffers(*pagedStream, /*outputTransient*/false); + } + } + // Flush paged streams to write any remaining buffered page data + for (auto& pair : pagedStreams) { + pair.second->flush(); + } +} + +} // namespace internal + +/// Per-grid-type codec-specific options for PointDataCodec +/// Contains point attribute filtering options +struct OPENVDB_API PointDataCodecTypeData : public io::ReadTypedOptions +{ + // Point Attribute Options - which attributes to read + std::vector pointAttributeNames; +}; // struct PointDataCodecTypeData + +template +struct PointDataCodec : public TopologyCodec +{ + using Ptr = std::unique_ptr>; + + ~PointDataCodec() noexcept = default; + + static inline std::string name() { return GridT::gridType(); } + + void readBuffers(std::istream& is, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) final + { + GridT& grid = static_cast(*data.grid); + + std::vector pointAttributeNames; + + // Look up point-specific options if provided + auto it = options.typeData.find(name()); + if (it != options.typeData.end()) { + auto& pointTypeData = io::ReadTypedOptions::cast(it->second); + pointAttributeNames = pointTypeData.pointAttributeNames; + } + + io::checkFormatVersion(is); + + bool saveFloatAsHalf = grid.saveFloatAsHalf(); + + auto& tree = grid.tree(); + tree.clearAllAccessors(); + + uint16_t numPasses = 1; + is.read(reinterpret_cast(&numPasses), sizeof(uint16_t)); + const Index attributes = (numPasses - 4) / 2; + + using LeafT = typename GridT::TreeType::LeafNodeType; + std::vector leaves; + tree.getNodes(leaves); + + // Pass 0: read voxel data sizes + std::map voxelBufferSizes; + internal::readPointDataVoxelSizes(leaves, is, voxelBufferSizes); + + // Pass 1: read descriptor and attribute metadata + internal::readPointDataDescriptors(leaves, is); + + // Build set of attribute indices to skip based on pointAttributeNames. + // An empty pointAttributeNames means no filtering (read all attributes). + std::set skipIndices; + if (!pointAttributeNames.empty() && !leaves.empty()) { + const auto& nameMap = leaves[0]->attributeSet().descriptor().map(); + const std::set wantedNames( + pointAttributeNames.begin(), + pointAttributeNames.end()); + for (const auto& namePos : nameMap) { + if (wantedNames.find(namePos.first) == wantedNames.end()) { + skipIndices.insert(static_cast(namePos.second)); + } + } + } + + // Passes 2..N+1: read attribute buffer sizes + std::map pagedStreams; + for (Index i = 0; i < attributes; ++i) { + if (skipIndices.count(i)) { + internal::skipPointDataAttributeSizes(leaves, is, i, pagedStreams); + } else { + internal::readPointDataAttributeSizes(leaves, is, i, pagedStreams); + } + } + + // Pass N+2: read voxel data + internal::readPointDataVoxelData(leaves, is, saveFloatAsHalf, + tree.background(), voxelBufferSizes); + + // Passes N+3..2N+2: read attribute data buffers + for (Index i = 0; i < attributes; ++i) { + if (skipIndices.count(i)) { + internal::skipPointDataAttributeData(leaves, is, i, pagedStreams); + } else { + internal::readPointDataAttributeData(leaves, is, i, pagedStreams); + } + } + + // Drop skipped attributes from each leaf's AttributeSet + if (!skipIndices.empty() && !leaves.empty()) { + const std::vector dropPositions(skipIndices.begin(), skipIndices.end()); + auto filteredDescriptor = + leaves[0]->attributeSet().descriptorPtr()->duplicateDrop(dropPositions); + for (auto* leaf : leaves) { + leaf->dropAttributes(dropPositions, + leaf->attributeSet().descriptor(), filteredDescriptor); + } + } + + // PointDataGrid uses multiple passes, so clip after reading + // the buffers if bbox is not infinite. + if (options.clipBBox.isSorted()) { + CoordBBox indexBBox = + grid.constTransform().worldToIndexNodeCentered(options.clipBBox); + grid.tree().root().clip(indexBBox); + } + } + + void writeBuffers(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) final + { + const GridT& grid = static_cast(gridBase); + bool saveFloatAsHalf = grid.saveFloatAsHalf(); + + using LeafT = typename GridT::TreeType::LeafNodeType; + std::vector leaves; + grid.tree().getNodes(leaves); + + // Determine how many leaf buffer passes are required for this grid + uint16_t numPasses = + static_cast(internal::countPointDataPasses(leaves)); + os.write(reinterpret_cast(&numPasses), sizeof(uint16_t)); + + const Index attributes = (numPasses - 4) / 2; + + // Pass 0: write voxel data sizes + descriptor tracking + bool matching = true; + points::AttributeSet::Descriptor::Ptr sharedDescriptor; + internal::writePointDataVoxelSizes(leaves, os, matching, sharedDescriptor); + + // Pass 1: write descriptor and attribute metadata + internal::writePointDataDescriptors(leaves, os, matching, sharedDescriptor); + + // Passes 2..N+1: write attribute buffer sizes (page headers) + for (Index i = 0; i < attributes; ++i) { + internal::writePointDataAttributeSizes(leaves, os, i); + } + + // Pass N+2: write voxel data + internal::writePointDataVoxelData(leaves, os, saveFloatAsHalf); + + // Passes N+3..2N+2: write attribute data buffers (page data) + for (Index i = 0; i < attributes; ++i) { + internal::writePointDataAttributeData(leaves, os, i); + } + } +}; // struct PointDataCodec + +} // namespace codecs +} // namespace OPENVDB_VERSION_NAME +} // namespace openvdb + +#endif // OPENVDB_IO_CODECS_POINTDATACODEC_HAS_BEEN_INCLUDED diff --git a/openvdb/openvdb/io/Codec.cc b/openvdb/openvdb/io/Codec.cc index 3e437fab17..4873f779aa 100644 --- a/openvdb/openvdb/io/Codec.cc +++ b/openvdb/openvdb/io/Codec.cc @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -60,6 +61,7 @@ void initialize() CodecRegistry::registerCodec>(); CodecRegistry::registerCodec>(); CodecRegistry::registerCodec>(); + CodecRegistry::registerCodec>(); // register the plugin that converts from scalar to mask/bool NumericGridTypes::foreach(); diff --git a/openvdb/openvdb/points/AttributeArray.h b/openvdb/openvdb/points/AttributeArray.h index 3ad2d3a48d..23ff8f8149 100644 --- a/openvdb/openvdb/points/AttributeArray.h +++ b/openvdb/openvdb/points/AttributeArray.h @@ -308,6 +308,9 @@ class OPENVDB_API AttributeArray /// Read attribute buffers from a paged stream. virtual void readPagedBuffers(compression::PagedInputStream&) = 0; + /// Skip attribute buffers in a paged stream without reading or + /// allocating any data. + void skipPagedBuffers(compression::PagedInputStream&); /// Write attribute buffers to a paged stream. /// @param outputTransient if true, write out transient attributes virtual void writePagedBuffers(compression::PagedOutputStream&, bool outputTransient) const = 0; @@ -1599,6 +1602,43 @@ TypedAttributeArray::readPagedBuffers(compression::PagedInpu } +inline void +AttributeArray::skipPagedBuffers(compression::PagedInputStream& is) +{ + if (!mUsePagedRead) { + if (!is.sizeOnly()) { + // for non-paged data, seek past the raw data in the stream + std::istream& inputStream = is.getInputStream(); + uint8_t bloscCompressed(0); + if (!mIsUniform) inputStream.read(reinterpret_cast(&bloscCompressed), sizeof(uint8_t)); + inputStream.seekg(mCompressedBytes, std::ios_base::cur); + mCompressedBytes = 0; + mFlags = static_cast(mFlags & ~PARTIALREAD); + } + return; + } + + if (is.sizeOnly()) + { + size_t compressedBytes(mCompressedBytes); + mCompressedBytes = 0; + mFlags = static_cast(mFlags & ~PARTIALREAD); + OPENVDB_ASSERT(!mPageHandle); + mPageHandle = is.createHandle(compressedBytes); + return; + } + + OPENVDB_ASSERT(mPageHandle); + + is.skip(mPageHandle, std::streamsize(mPageHandle->size())); + mPageHandle.reset(); + + // clear page state + + mUsePagedRead = 0; +} + + template void TypedAttributeArray::write(std::ostream& os) const diff --git a/openvdb/openvdb/points/StreamCompression.cc b/openvdb/openvdb/points/StreamCompression.cc index 5760f3b4ae..f18d016463 100644 --- a/openvdb/openvdb/points/StreamCompression.cc +++ b/openvdb/openvdb/points/StreamCompression.cc @@ -338,6 +338,21 @@ Page::readBuffers(std::istream&is, bool delayed) } +void +Page::skipBuffers(std::istream& is) +{ + OPENVDB_ASSERT(mInfo); + + bool isCompressed = mInfo->compressedBytes > 0; + std::streamsize bytes = isCompressed ? + mInfo->compressedBytes : -mInfo->compressedBytes; + + is.seekg(bytes, std::ios_base::cur); + + mInfo.reset(); +} + + void Page::copy(const std::unique_ptr& temp, int pageSize) { @@ -438,6 +453,23 @@ PagedInputStream::read(PageHandle::Ptr& pageHandle, std::streamsize n, bool dela } +void +PagedInputStream::skip(PageHandle::Ptr& pageHandle, std::streamsize n) +{ + OPENVDB_ASSERT(mByteIndex <= mUncompressedBytes); + + Page& page = pageHandle->page(); + + if (mByteIndex == mUncompressedBytes) { + mUncompressedBytes = static_cast(page.uncompressedBytes()); + page.skipBuffers(*mIs); + mByteIndex = 0; + } + + mByteIndex += int(n); +} + + //////////////////////////////////////// diff --git a/openvdb/openvdb/points/StreamCompression.h b/openvdb/openvdb/points/StreamCompression.h index d983ea2404..2efbcda35c 100644 --- a/openvdb/openvdb/points/StreamCompression.h +++ b/openvdb/openvdb/points/StreamCompression.h @@ -142,6 +142,10 @@ class OPENVDB_API Page /// pointers will be stored to load the data lazily. void readBuffers(std::istream&, bool delayed); + /// @brief Skip the Page buffers by seeking past the compressed data + /// without reading or decompressing it. + void skipBuffers(std::istream&); + OPENVDB_DEPRECATED_MESSAGE("Always returns false. This method is deprecated and will be removed. Delayed loading is no longer supported.") bool isOutOfCore() const { return false; } @@ -222,6 +226,10 @@ class OPENVDB_API PagedInputStream /// an immediate read of the data. void read(PageHandle::Ptr& pageHandle, std::streamsize n, bool delayed = true); + /// @brief Skip past the page data referenced by @a pageHandle without + /// reading or decompressing it. + void skip(PageHandle::Ptr& pageHandle, std::streamsize n); + private: int mByteIndex = 0; int mUncompressedBytes = 0; diff --git a/openvdb/openvdb/unittest/TestPointCodec.cc b/openvdb/openvdb/unittest/TestPointCodec.cc index 2b8911419e..81f8ff2358 100644 --- a/openvdb/openvdb/unittest/TestPointCodec.cc +++ b/openvdb/openvdb/unittest/TestPointCodec.cc @@ -5,6 +5,7 @@ #include #include #include +#include #include #include "util.h" // for unittest_util::genPoints @@ -145,3 +146,402 @@ TEST_F(TestPointCodec, testPointIndexCodecIO) CodecRegistry::clear(); std::remove(codecPath.c_str()); } + +TEST_F(TestPointCodec, testPointDataCodecIO) +{ + using namespace openvdb; + using namespace openvdb::io; + using namespace openvdb::points; + using PointDataTree = PointDataGrid::TreeType; + + openvdb::initialize(); + CodecRegistry::clear(); + + // Helper: compare P attribute values leaf-by-leaf between two PointDataGrids + auto comparePositions = [](const PointDataGrid& a, const PointDataGrid& b) { + auto aIt = a.tree().cbeginLeaf(); + auto bIt = b.tree().cbeginLeaf(); + for (; aIt && bIt; ++aIt, ++bIt) { + EXPECT_EQ(aIt->pointCount(), bIt->pointCount()); + AttributeHandle aH(aIt->constAttributeArray("P")); + AttributeHandle bH(bIt->constAttributeArray("P")); + for (Index i = 0; i < aIt->pointCount(); ++i) { + const Vec3f av = aH.get(i); + const Vec3f bv = bH.get(i); + EXPECT_NEAR(av.x(), bv.x(), 1e-6f); + EXPECT_NEAR(av.y(), bv.y(), 1e-6f); + EXPECT_NEAR(av.z(), bv.z(), 1e-6f); + } + } + EXPECT_TRUE(!aIt && !bIt); + }; + + // ----------------------------------------------------------------------- + // Section A: Positions only + // ----------------------------------------------------------------------- + { + const std::vector positions = { + Vec3f(0.0f, 1.0f, 0.0f), + Vec3f(1.5f, 3.5f, 1.0f), + Vec3f(-1.0f, 6.0f, -2.0f), + Vec3f(1.1f, 1.25f, 0.06f) + }; + + const float voxelSize = 0.5f; + math::Transform::Ptr transform = math::Transform::createLinearTransform(voxelSize); + + PointDataGrid::Ptr srcGrid = + createPointDataGrid(positions, *transform); + srcGrid->setName("pdg_positions"); + + const std::string rawPath = "testPDG_A_raw.vdb"; + + // Phase 1: write/read without codec + { + io::File f(rawPath); + f.write(GridPtrVec{srcGrid}); + } + + PointDataGrid::Ptr rawGrid; + { + io::File f(rawPath); + f.open(); + rawGrid = gridPtrCast(f.readGrid("pdg_positions")); + f.close(); + } + ASSERT_TRUE(rawGrid); + EXPECT_TRUE(srcGrid->tree().hasSameTopology(rawGrid->tree())); + + PointDataGrid::Ptr rawTopo; + { + io::File f(rawPath); + f.open(); + GridBase::Ptr base; + EXPECT_NO_THROW(base = f.readGrid("pdg_positions")); + rawTopo = gridPtrCast(base); + f.close(); + } + ASSERT_TRUE(rawTopo); + EXPECT_EQ(rawTopo->activeVoxelCount(), Index64(4)); + + std::remove(rawPath.c_str()); + + const std::string codecPath = "testPDG_A_codec.vdb"; + + // Phase 2: register codec, write/read with codec + io::internal::initialize(); + ASSERT_TRUE(CodecRegistry::isRegistered(PointDataGrid::gridType())); + + { + io::File f(codecPath); + f.write(GridPtrVec{srcGrid}); + } + + PointDataGrid::Ptr codecGrid; + { + io::File f(codecPath); + f.open(); + codecGrid = gridPtrCast(f.readGrid("pdg_positions")); + f.close(); + } + ASSERT_TRUE(codecGrid); + + // Phase 3: compare src vs codec + EXPECT_TRUE(srcGrid->tree().hasSameTopology(codecGrid->tree())); + EXPECT_EQ(pointCount(srcGrid->tree()), pointCount(codecGrid->tree())); + comparePositions(*srcGrid, *codecGrid); + + // Phase 4: TopologyOnly read + ReadOptions topoOpts; + topoOpts.readMode = ReadMode::TopologyOnly; + + PointDataGrid::Ptr codecTopo; + { + io::File f(codecPath); + f.open(); + GridBase::Ptr base; + EXPECT_NO_THROW(base = f.readGrid("pdg_positions", topoOpts)); + codecTopo = gridPtrCast(base); + f.close(); + } + ASSERT_TRUE(codecTopo); + EXPECT_EQ(codecTopo->activeVoxelCount(), Index64(0)); + EXPECT_TRUE(codecTopo->tree().leafCount() == 0); + + std::remove(codecPath.c_str()); + } + + // ----------------------------------------------------------------------- + // Section B: Multiple attributes + // ----------------------------------------------------------------------- + { + const std::vector positions = { + Vec3f(0.0f, 1.0f, 0.0f), + Vec3f(1.5f, 3.5f, 1.0f), + Vec3f(-1.0f, 6.0f, -2.0f), + Vec3f(1.1f, 1.25f, 0.06f) + }; + const std::vector velocities = { + Vec3f(1.0f, 0.0f, 0.0f), + Vec3f(0.0f, 1.0f, 0.0f), + Vec3f(0.0f, 0.0f, 1.0f), + Vec3f(1.0f, 1.0f, 0.5f) + }; + const std::vector ids = {0, 1, 2, 3}; + + const float voxelSize = 0.5f; + math::Transform::Ptr transform = math::Transform::createLinearTransform(voxelSize); + + PointAttributeVector posWrapper(positions); + tools::PointIndexGrid::Ptr pointIndexGrid = + tools::createPointIndexGrid(posWrapper, *transform); + + PointDataGrid::Ptr srcGrid = + createPointDataGrid( + *pointIndexGrid, posWrapper, *transform); + srcGrid->setName("pdg_multi"); + + PointDataTree& tree = srcGrid->tree(); + tools::PointIndexTree& indexTree = pointIndexGrid->tree(); + + appendAttribute(tree, "velocity"); + populateAttribute>( + tree, indexTree, "velocity", + PointAttributeVector(velocities)); + + appendAttribute(tree, "id"); + populateAttribute>( + tree, indexTree, "id", + PointAttributeVector(ids)); + + // Verify attribute count on src grid (P, velocity, id) + { + auto leafIt = srcGrid->tree().cbeginLeaf(); + ASSERT_TRUE(leafIt); + EXPECT_EQ(leafIt->attributeSet().size(), size_t(3)); + } + + CodecRegistry::clear(); + + const std::string rawPath = "testPDG_B_raw.vdb"; + + // Phase 1: write/read without codec + { + io::File f(rawPath); + f.write(GridPtrVec{srcGrid}); + } + + PointDataGrid::Ptr rawGrid; + { + io::File f(rawPath); + f.open(); + rawGrid = gridPtrCast(f.readGrid("pdg_multi")); + f.close(); + } + ASSERT_TRUE(rawGrid); + + std::remove(rawPath.c_str()); + + const std::string codecPath = "testPDG_B_codec.vdb"; + + // Phase 2: register codec, write/read with codec + io::internal::initialize(); + + { + io::File f(codecPath); + f.write(GridPtrVec{srcGrid}); + } + + PointDataGrid::Ptr codecGrid; + { + io::File f(codecPath); + f.open(); + codecGrid = gridPtrCast(f.readGrid("pdg_multi")); + f.close(); + } + ASSERT_TRUE(codecGrid); + + EXPECT_TRUE(srcGrid->tree().hasSameTopology(codecGrid->tree())); + EXPECT_EQ(pointCount(srcGrid->tree()), pointCount(codecGrid->tree())); + + // Verify attribute count on codec grid + { + auto leafIt = codecGrid->tree().cbeginLeaf(); + ASSERT_TRUE(leafIt); + EXPECT_EQ(leafIt->attributeSet().size(), size_t(3)); + } + + // Compare all three attributes leaf-by-leaf + { + auto srcIt = srcGrid->tree().cbeginLeaf(); + auto codecIt = codecGrid->tree().cbeginLeaf(); + for (; srcIt && codecIt; ++srcIt, ++codecIt) { + EXPECT_EQ(srcIt->pointCount(), codecIt->pointCount()); + AttributeHandle srcP(srcIt->constAttributeArray("P")); + AttributeHandle codecP(codecIt->constAttributeArray("P")); + AttributeHandle srcVel(srcIt->constAttributeArray("velocity")); + AttributeHandle codecVel(codecIt->constAttributeArray("velocity")); + AttributeHandle srcId(srcIt->constAttributeArray("id")); + AttributeHandle codecId(codecIt->constAttributeArray("id")); + for (Index i = 0; i < srcIt->pointCount(); ++i) { + const Vec3f rp = srcP.get(i); + const Vec3f cp = codecP.get(i); + EXPECT_NEAR(rp.x(), cp.x(), 1e-6f); + EXPECT_NEAR(rp.y(), cp.y(), 1e-6f); + EXPECT_NEAR(rp.z(), cp.z(), 1e-6f); + const Vec3f rv = srcVel.get(i); + const Vec3f cv = codecVel.get(i); + EXPECT_NEAR(rv.x(), cv.x(), 1e-6f); + EXPECT_NEAR(rv.y(), cv.y(), 1e-6f); + EXPECT_NEAR(rv.z(), cv.z(), 1e-6f); + EXPECT_EQ(srcId.get(i), codecId.get(i)); + } + } + EXPECT_TRUE(!srcIt && !codecIt); + } + + CodecRegistry::clear(); + std::remove(codecPath.c_str()); + } + + // ----------------------------------------------------------------------- + // Section C: Shared vs non-shared descriptors + // ----------------------------------------------------------------------- + { + std::vector pts; + unittest_util::genPoints(100, pts); + + std::vector positions; + positions.reserve(pts.size()); + for (const auto& p : pts) { + positions.emplace_back(float(p.x()), float(p.y()), float(p.z())); + } + + const double voxelSize = 0.1; + math::Transform::Ptr transform = math::Transform::createLinearTransform(voxelSize); + + PointDataGrid::Ptr srcGrid = + createPointDataGrid(positions, *transform); + srcGrid->setName("pdg_desc"); + + // All leaves should share one Descriptor::Ptr initially + ASSERT_GT(srcGrid->tree().leafCount(), Index32(1)); + { + auto leafIt = srcGrid->tree().cbeginLeaf(); + auto firstDescPtr = leafIt->attributeSet().descriptorPtr(); + for (; leafIt; ++leafIt) { + EXPECT_EQ(leafIt->attributeSet().descriptorPtr(), firstDescPtr); + } + } + + io::internal::initialize(); + + // -- C1: Shared descriptors (default) -- + // All leaves already share one pointer; this exercises the header=1 write path. + const std::string sharedPath = "testPDG_C_shared.vdb"; + { + io::File f(sharedPath); + f.write(GridPtrVec{srcGrid}); + } + + PointDataGrid::Ptr sharedGrid; + { + io::File f(sharedPath); + f.open(); + sharedGrid = gridPtrCast(f.readGrid("pdg_desc")); + f.close(); + } + ASSERT_TRUE(sharedGrid); + EXPECT_TRUE(srcGrid->tree().hasSameTopology(sharedGrid->tree())); + EXPECT_EQ(pointCount(srcGrid->tree()), pointCount(sharedGrid->tree())); + comparePositions(*srcGrid, *sharedGrid); + + // -- C2: After makeDescriptorUnique -- + // makeDescriptorUnique() creates ONE new descriptor and assigns it to every + // leaf, so all leaves still share a single pointer (the new copy). + // The codec still detects matching descriptors and writes header=1; + // this is a regression check that the round-trip remains correct. + makeDescriptorUnique(srcGrid->tree()); + { + auto leafIt = srcGrid->tree().cbeginLeaf(); + auto firstDescPtr = leafIt->attributeSet().descriptorPtr(); + ++leafIt; + if (leafIt) { + // All leaves share the same new pointer + EXPECT_EQ(leafIt->attributeSet().descriptorPtr(), firstDescPtr); + } + } + + const std::string nonSharedPath = "testPDG_C_nonshared.vdb"; + { + io::File f(nonSharedPath); + f.write(GridPtrVec{srcGrid}); + } + + PointDataGrid::Ptr nonSharedGrid; + { + io::File f(nonSharedPath); + f.open(); + nonSharedGrid = gridPtrCast(f.readGrid("pdg_desc")); + f.close(); + } + ASSERT_TRUE(nonSharedGrid); + EXPECT_TRUE(srcGrid->tree().hasSameTopology(nonSharedGrid->tree())); + EXPECT_EQ(pointCount(srcGrid->tree()), pointCount(nonSharedGrid->tree())); + comparePositions(*sharedGrid, *nonSharedGrid); + + // -- C3: Genuinely different descriptors (exercises the header=0 write path) -- + // Add "extra" to all leaves, then drop it from only the first leaf so that + // leaf descriptors differ by value, triggering matching=false in the codec. + appendAttribute(srcGrid->tree(), "extra"); + makeDescriptorUnique(srcGrid->tree()); + srcGrid->setName("pdg_desc_diff"); + + { + auto leafIt = srcGrid->tree().beginLeaf(); + ASSERT_TRUE(leafIt); + const size_t extraIdx = + leafIt->attributeSet().descriptor().find("extra"); + ASSERT_NE(extraIdx, AttributeSet::INVALID_POS); + const std::vector dropIndices = {extraIdx}; + AttributeSet::Descriptor::Ptr newDesc = + leafIt->attributeSet().descriptor().duplicateDrop(dropIndices); + leafIt->dropAttributes( + dropIndices, leafIt->attributeSet().descriptor(), newDesc); + } + + const std::string diffPath = "testPDG_C_diff.vdb"; + { + io::File f(diffPath); + f.write(GridPtrVec{srcGrid}); + } + + PointDataGrid::Ptr diffGrid; + { + io::File f(diffPath); + f.open(); + diffGrid = gridPtrCast(f.readGrid("pdg_desc_diff")); + f.close(); + } + ASSERT_TRUE(diffGrid); + EXPECT_TRUE(srcGrid->tree().hasSameTopology(diffGrid->tree())); + EXPECT_EQ(pointCount(srcGrid->tree()), pointCount(diffGrid->tree())); + + // First leaf has {P} only; remaining leaves have {P, extra} + { + auto diffIt = diffGrid->tree().cbeginLeaf(); + ASSERT_TRUE(diffIt); + EXPECT_EQ(diffIt->attributeSet().size(), size_t(1)); + ++diffIt; + if (diffIt) { + EXPECT_EQ(diffIt->attributeSet().size(), size_t(2)); + } + } + + std::remove(sharedPath.c_str()); + std::remove(nonSharedPath.c_str()); + std::remove(diffPath.c_str()); + } +} From 056b3223cc203637b5417ed22654dc20dec7b697 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Thu, 14 May 2026 14:29:07 -0700 Subject: [PATCH 7/8] Address feedback Signed-off-by: Dan Bailey --- openvdb/openvdb/codecs/PointDataCodec.h | 18 ++++++++++-------- openvdb/openvdb/codecs/PointIndexCodec.h | 6 ++++-- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/openvdb/openvdb/codecs/PointDataCodec.h b/openvdb/openvdb/codecs/PointDataCodec.h index e5cc550fe4..8ad1d829db 100644 --- a/openvdb/openvdb/codecs/PointDataCodec.h +++ b/openvdb/openvdb/codecs/PointDataCodec.h @@ -12,7 +12,7 @@ #include #include -#include "ScalarLeafCodec.h" +#include "impl/ScalarLeafCodec.h" #include "TopologyCodec.h" #include @@ -32,8 +32,9 @@ inline typename PagedStreamPtrT::element_type* getOrCreatePagedStream( auto it = pagedStreams.find(attributeIndex); if (it != pagedStreams.end()) return it->second.get(); using PagedStreamT = typename PagedStreamPtrT::element_type; - pagedStreams[attributeIndex] = std::make_shared(); - return pagedStreams[attributeIndex].get(); + auto ptr = std::make_shared(); + auto& stored = (pagedStreams[attributeIndex] = std::move(ptr)); + return stored.get(); } //////////////////////////////////////// @@ -41,7 +42,7 @@ inline typename PagedStreamPtrT::element_type* getOrCreatePagedStream( template inline void readPointDataVoxelSizes(const std::vector& leaves, - std::istream& is, std::map& voxelBufferSizes) + std::istream& is, std::unordered_map& voxelBufferSizes) { for (auto* leaf : leaves) { uint16_t voxelBufferSize; @@ -110,9 +111,8 @@ template inline void readPointDataVoxelData(const std::vector& leaves, std::istream& is, bool saveFloatAsHalf, const typename LeafT::ValueType& background, - const std::map& voxelBufferSizes) + const std::unordered_map& voxelBufferSizes) { - (void) voxelBufferSizes; using BaseLeaf = typename LeafT::BaseLeaf; for (auto* leaf : leaves) { OPENVDB_ASSERT(voxelBufferSizes.find(leaf->origin()) != voxelBufferSizes.end()); @@ -302,7 +302,7 @@ struct OPENVDB_API PointDataCodecTypeData : public io::ReadTypedOptions }; // struct PointDataCodecTypeData template -struct PointDataCodec : public TopologyCodec +struct PointDataCodec final: public TopologyCodec { using Ptr = std::unique_ptr>; @@ -312,6 +312,8 @@ struct PointDataCodec : public TopologyCodec void readBuffers(std::istream& is, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) final { + OPENVDB_ASSERT(dynamic_cast(data.grid.get())); + GridT& grid = static_cast(*data.grid); std::vector pointAttributeNames; @@ -339,7 +341,7 @@ struct PointDataCodec : public TopologyCodec tree.getNodes(leaves); // Pass 0: read voxel data sizes - std::map voxelBufferSizes; + std::unordered_map voxelBufferSizes; internal::readPointDataVoxelSizes(leaves, is, voxelBufferSizes); // Pass 1: read descriptor and attribute metadata diff --git a/openvdb/openvdb/codecs/PointIndexCodec.h b/openvdb/openvdb/codecs/PointIndexCodec.h index d8d9f1ee32..851ee140fd 100644 --- a/openvdb/openvdb/codecs/PointIndexCodec.h +++ b/openvdb/openvdb/codecs/PointIndexCodec.h @@ -8,7 +8,7 @@ #include -#include "ScalarLeafCodec.h" +#include "impl/ScalarLeafCodec.h" #include "TopologyCodec.h" namespace openvdb { @@ -106,7 +106,7 @@ struct WritePointIndexBuffersOp } // namespace internal template -struct PointIndexCodec : public TopologyCodec +struct PointIndexCodec final: public TopologyCodec { using Ptr = std::unique_ptr>; @@ -116,6 +116,8 @@ struct PointIndexCodec : public TopologyCodec void readBuffers(std::istream& is, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics& diagnostics) final { + OPENVDB_ASSERT(dynamic_cast(data.grid.get())); + GridT& grid = static_cast(*data.grid); if (grid.hasMultiPassIO()) { From 031489e38980ea92a57ef0f69df08a44d6994567 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Thu, 14 May 2026 15:01:25 -0700 Subject: [PATCH 8/8] Address feedback (fix unused error) Signed-off-by: Dan Bailey --- openvdb/openvdb/codecs/PointDataCodec.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openvdb/openvdb/codecs/PointDataCodec.h b/openvdb/openvdb/codecs/PointDataCodec.h index 8ad1d829db..f4bc713bd7 100644 --- a/openvdb/openvdb/codecs/PointDataCodec.h +++ b/openvdb/openvdb/codecs/PointDataCodec.h @@ -111,7 +111,7 @@ template inline void readPointDataVoxelData(const std::vector& leaves, std::istream& is, bool saveFloatAsHalf, const typename LeafT::ValueType& background, - const std::unordered_map& voxelBufferSizes) + [[maybe_unused]] const std::unordered_map& voxelBufferSizes) { using BaseLeaf = typename LeafT::BaseLeaf; for (auto* leaf : leaves) {