diff --git a/cpp/open3d/t/geometry/PointCloud.cpp b/cpp/open3d/t/geometry/PointCloud.cpp index 4ab5b87e58c..2e057ce3e91 100644 --- a/cpp/open3d/t/geometry/PointCloud.cpp +++ b/cpp/open3d/t/geometry/PointCloud.cpp @@ -1370,20 +1370,38 @@ core::Tensor PointCloud::ComputeMetrics(const PointCloud &pcd2, } bool PointCloud::IsGaussianSplat() const { - auto num_points = GetPointPositions().GetLength(); bool have_all_attrs = HasPointAttr("opacity") && HasPointAttr("rot") && HasPointAttr("scale") && HasPointAttr("f_dc"); if (!have_all_attrs) { // not 3DGS, no messages. return false; } // Existing but invalid attributes cause errors. + auto num_points = GetPointPositions().GetLength(); core::AssertTensorShape(GetPointAttr("opacity"), {num_points, 1}); core::AssertTensorShape(GetPointAttr("rot"), {num_points, 4}); core::AssertTensorShape(GetPointAttr("scale"), {num_points, 3}); core::AssertTensorShape(GetPointAttr("f_dc"), {num_points, 3}); - // GaussianSplatGetSHOrder(); // TODO: Tests f_rest shape is valid. + GaussianSplatGetSHOrder(); // Tests f_rest shape is valid. return true; } + +int PointCloud::GaussianSplatGetSHOrder() const { + if (point_attr_.find("f_rest") == point_attr_.end()) { + return 0; + } + const core::Tensor &f_rest = GetPointAttr("f_rest"); + auto num_points = GetPointPositions().GetLength(); + core::AssertTensorShape(f_rest, {num_points, core::None, 3}); + auto Nc = f_rest.GetShape(1); + auto degp1 = static_cast(sqrt(Nc + 1)); + if (degp1 * degp1 != Nc + 1) { + utility::LogError( + "f_rest has incomplete Spherical Harmonics coefficients " + "({}), expected 0, 3, 8 or 15.", + Nc); + } + return degp1 - 1; +} } // namespace geometry } // namespace t } // namespace open3d diff --git a/cpp/open3d/t/geometry/PointCloud.h b/cpp/open3d/t/geometry/PointCloud.h index 9d15fda0037..d4ec3afb112 100644 --- a/cpp/open3d/t/geometry/PointCloud.h +++ b/cpp/open3d/t/geometry/PointCloud.h @@ -746,6 +746,12 @@ class PointCloud : public Geometry, public DrawableGeometry { /// shape). bool IsGaussianSplat() const; + /// \brief Returns the order of spherical harmonics used for Gaussian + /// Splatting. Returns 0 if f_rest is not present. + /// \throws If point cloud has f_rest 3DGS attribute, with the wrong + /// shape. + int GaussianSplatGetSHOrder() const; + protected: core::Device device_ = core::Device("CPU:0"); TensorMap point_attr_; diff --git a/cpp/open3d/t/io/file_format/FileSPLAT.cpp b/cpp/open3d/t/io/file_format/FileSPLAT.cpp index 89d3301b12a..5084212399f 100644 --- a/cpp/open3d/t/io/file_format/FileSPLAT.cpp +++ b/cpp/open3d/t/io/file_format/FileSPLAT.cpp @@ -76,7 +76,6 @@ std::vector SortedSplatIndices(geometry::TensorMap &t_map) { return score_left < score_right; // Sort in descending order }); - return indices; } @@ -179,7 +178,7 @@ bool ReadPointCloudFromSPLAT(const std::string &filename, quat_norm += rot_float[i] * rot_float[i]; } quat_norm = sqrt(quat_norm); - if (quat_norm > 1e-6) { + if (quat_norm > std::numeric_limits::epsilon()) { for (int i = 0; i < 4; i++) { rot_float[i] /= quat_norm; } @@ -199,7 +198,7 @@ bool ReadPointCloudFromSPLAT(const std::string &filename, reporter.Finish(); return true; } catch (const std::exception &e) { - utility::LogError("Read SPLAT failed: {}", e.what()); + utility::LogError("Read SPLAT file {} failed: {}", filename, e.what()); } return false; } @@ -251,8 +250,8 @@ bool WritePointCloudToSPLAT(const std::string &filename, splat_file.exceptions(std::ofstream::badbit); // failbit not set for // binary IO errors } catch (const std::ios_base::failure &) { - utility::LogError("Write SPLAT failed: unable to open file: {}.", - filename); + utility::LogWarning("Write SPLAT failed: unable to open file: {}.", + filename); return false; } @@ -287,13 +286,19 @@ bool WritePointCloudToSPLAT(const std::string &filename, Eigen::Vector4f rot{rot_ptr[rot_offset], rot_ptr[rot_offset + 1], rot_ptr[rot_offset + 2], rot_ptr[rot_offset + 3]}; + if (auto quat_norm = rot.norm(); + quat_norm > std::numeric_limits::epsilon()) { + rot /= quat_norm; + } else { + rot = {1.f, 0.f, 0.f, 0.f}; // wxyz quaternion + } // offset should be 127, but we follow the reference // antimatter/convert.py code - rot = (rot.normalized() * 128.0).array().round() + 128.0; - auto int8_rot = - rot.cwiseMin(255.0).cwiseMax(0.0).cast().eval(); - splat_file.write(reinterpret_cast(int8_rot.data()), - 4 * sizeof(int8_t)); + rot = (rot * 128.0).array().round() + 128.0; + auto uint8_rot = + rot.cwiseMin(255.0).cwiseMax(0.0).cast().eval(); + splat_file.write(reinterpret_cast(uint8_rot.data()), + 4 * sizeof(uint8_t)); if (i % 1000 == 0) { reporter.Update(i); @@ -303,7 +308,8 @@ bool WritePointCloudToSPLAT(const std::string &filename, reporter.Finish(); return true; } catch (const std::ios_base::failure &e) { - utility::LogError("Write SPLAT failed: {}", e.what()); + utility::LogWarning("Write SPLAT to file {} failed: {}", filename, + e.what()); return false; } } diff --git a/cpp/open3d/visualization/CMakeLists.txt b/cpp/open3d/visualization/CMakeLists.txt index 8cf18c2088c..833ca2ae40f 100644 --- a/cpp/open3d/visualization/CMakeLists.txt +++ b/cpp/open3d/visualization/CMakeLists.txt @@ -80,6 +80,7 @@ if (BUILD_GUI) rendering/filament/LineSetBuffers.cpp rendering/filament/PointCloudBuffers.cpp rendering/filament/TriangleMeshBuffers.cpp + rendering/filament/GaussianSplatBuffers.cpp ) target_sources(visualization PRIVATE diff --git a/cpp/open3d/visualization/rendering/MaterialRecord.h b/cpp/open3d/visualization/rendering/MaterialRecord.h index 6822cbaf1ff..ef1b174d35a 100644 --- a/cpp/open3d/visualization/rendering/MaterialRecord.h +++ b/cpp/open3d/visualization/rendering/MaterialRecord.h @@ -83,6 +83,9 @@ struct MaterialRecord { // Infinite ground plane float ground_plane_axis = 0.f; // 0: XZ; >0: XY; <0: YZ + // This is only used in gaussian splat. + int sh_degree = 0; + // Generic material properties std::unordered_map generic_params; std::unordered_map generic_imgs; diff --git a/cpp/open3d/visualization/rendering/filament/FilamentGeometryBuffersBuilder.cpp b/cpp/open3d/visualization/rendering/filament/FilamentGeometryBuffersBuilder.cpp index fe3dbd39bf7..1a09943c699 100644 --- a/cpp/open3d/visualization/rendering/filament/FilamentGeometryBuffersBuilder.cpp +++ b/cpp/open3d/visualization/rendering/filament/FilamentGeometryBuffersBuilder.cpp @@ -250,9 +250,16 @@ std::unique_ptr GeometryBuffersBuilder::GetBuilder( using GT = t::geometry::Geometry::GeometryType; switch (geometry.GetGeometryType()) { - case GT::PointCloud: - return std::make_unique( - static_cast(geometry)); + case GT::PointCloud: { + const t::geometry::PointCloud& pointcloud = + static_cast(geometry); + if (pointcloud.IsGaussianSplat()) { + return std::make_unique( + pointcloud); + } else { + return std::make_unique(pointcloud); + } + } case GT::TriangleMesh: return std::make_unique( static_cast(geometry)); diff --git a/cpp/open3d/visualization/rendering/filament/FilamentGeometryBuffersBuilder.h b/cpp/open3d/visualization/rendering/filament/FilamentGeometryBuffersBuilder.h index afaa4a84831..40bbd32d349 100644 --- a/cpp/open3d/visualization/rendering/filament/FilamentGeometryBuffersBuilder.h +++ b/cpp/open3d/visualization/rendering/filament/FilamentGeometryBuffersBuilder.h @@ -172,10 +172,43 @@ class TPointCloudBuffersBuilder : public GeometryBuffersBuilder { Buffers ConstructBuffers() override; filament::Box ComputeAABB() override; -private: +protected: t::geometry::PointCloud geometry_; }; +class TGaussianSplatBuffersBuilder : public TPointCloudBuffersBuilder { +public: + /// \brief Constructs a TGaussianSplatBuffersBuilder object. + /// + /// Initializes the Gaussian Splat buffers from the provided \p geometry and + /// ensures that all necessary attributes are present and correctly + /// formatted. If the geometry is not a Gaussian Splat, a warning is issued. + /// Additionally, attributes like "f_dc", "opacity", "rot", "scale", and + /// "f_rest" are checked for their data type, and converted to Float32 if + /// they are not already in that format. + explicit TGaussianSplatBuffersBuilder( + const t::geometry::PointCloud& geometry); + + /// \brief Constructs vertex and index buffers for Gaussian Splat rendering. + /// + /// This function creates and configures GPU buffers to represent a Gaussian + /// Splat point cloud. It extracts attributes like positions, colors, + /// rotation, scale, and spherical harmonics coefficients from the provided + /// \ref geometry_ and organizes them into separate vertex buffer + /// attributes. + /// + /// The vertex buffer contains the following attributes: + /// - POSITION: Vertex positions (FLOAT3) + /// - COLOR: DC component and opacity (FLOAT4) + /// - TANGENTS: Rotation quaternion (FLOAT4) + /// - CUSTOM0: Scale (FLOAT4) + /// - CUSTOM1 to CUSTOM6: SH coefficients (FLOAT4) + /// + /// Each attribute is checked and converted to the expected data type if + /// necessary, and missing attributes are initialized with default values. + Buffers ConstructBuffers() override; +}; + class TLineSetBuffersBuilder : public GeometryBuffersBuilder { public: explicit TLineSetBuffersBuilder(const t::geometry::LineSet& geometry); diff --git a/cpp/open3d/visualization/rendering/filament/FilamentResourceManager.cpp b/cpp/open3d/visualization/rendering/filament/FilamentResourceManager.cpp index baee308e93c..dd10e441f59 100644 --- a/cpp/open3d/visualization/rendering/filament/FilamentResourceManager.cpp +++ b/cpp/open3d/visualization/rendering/filament/FilamentResourceManager.cpp @@ -318,6 +318,8 @@ TextureSettings GetSettingsFromImage(const t::geometry::Image& image, const MaterialHandle FilamentResourceManager::kDefaultLit = MaterialHandle::Next(); +const MaterialHandle FilamentResourceManager::kGaussianSplatShader = + MaterialHandle::Next(); const MaterialHandle FilamentResourceManager::kDefaultLitWithTransparency = MaterialHandle::Next(); const MaterialHandle FilamentResourceManager::kDefaultLitSSR = @@ -1017,6 +1019,28 @@ void FilamentResourceManager::LoadDefaults() { lit_mat->setDefaultParameter("anisotropyMap", texture, default_sampler); materials_[kDefaultLit] = BoxResource(lit_mat, engine_); + const auto gaussian_path = resource_root + "/gaussianSplat.filamat"; + auto gaussian_mat = LoadMaterialFromFile(gaussian_path, engine_); + gaussian_mat->setDefaultParameter("baseColor", filament::RgbType::sRGB, + default_color); + gaussian_mat->setDefaultParameter("baseRoughness", 0.7f); + gaussian_mat->setDefaultParameter("reflectance", 0.5f); + gaussian_mat->setDefaultParameter("baseMetallic", 0.f); + gaussian_mat->setDefaultParameter("clearCoat", 0.f); + gaussian_mat->setDefaultParameter("clearCoatRoughness", 0.f); + gaussian_mat->setDefaultParameter("anisotropy", 0.f); + gaussian_mat->setDefaultParameter("pointSize", 3.f); + gaussian_mat->setDefaultParameter("albedo", texture, default_sampler); + gaussian_mat->setDefaultParameter("ao_rough_metalMap", texture, + default_sampler); + gaussian_mat->setDefaultParameter("normalMap", normal_map, default_sampler); + gaussian_mat->setDefaultParameter("reflectanceMap", texture, + default_sampler); + + gaussian_mat->setDefaultParameter("anisotropyMap", texture, + default_sampler); + materials_[kGaussianSplatShader] = BoxResource(gaussian_mat, engine_); + const auto lit_trans_path = resource_root + "/defaultLitTransparency.filamat"; auto lit_trans_mat = LoadMaterialFromFile(lit_trans_path, engine_); diff --git a/cpp/open3d/visualization/rendering/filament/FilamentResourceManager.h b/cpp/open3d/visualization/rendering/filament/FilamentResourceManager.h index d3a6716bbab..5503306b8ae 100644 --- a/cpp/open3d/visualization/rendering/filament/FilamentResourceManager.h +++ b/cpp/open3d/visualization/rendering/filament/FilamentResourceManager.h @@ -50,6 +50,7 @@ namespace rendering { class FilamentResourceManager { public: static const MaterialHandle kDefaultLit; + static const MaterialHandle kGaussianSplatShader; static const MaterialHandle kDefaultLitWithTransparency; static const MaterialHandle kDefaultLitSSR; static const MaterialHandle kDefaultUnlit; diff --git a/cpp/open3d/visualization/rendering/filament/FilamentScene.cpp b/cpp/open3d/visualization/rendering/filament/FilamentScene.cpp index ecb6c5c89c0..0afb898538a 100644 --- a/cpp/open3d/visualization/rendering/filament/FilamentScene.cpp +++ b/cpp/open3d/visualization/rendering/filament/FilamentScene.cpp @@ -102,7 +102,8 @@ std::unordered_map shader_mappings = { ResourceManager::kDefaultUnlitPolygonOffsetShader}, {"unlitBackground", ResourceManager::kDefaultUnlitBackgroundShader}, {"infiniteGroundPlane", ResourceManager::kInfinitePlaneShader}, - {"unlitLine", ResourceManager::kDefaultLineShader}}; + {"unlitLine", ResourceManager::kDefaultLineShader}, + {"gaussianSplat", ResourceManager::kGaussianSplatShader}}; MaterialHandle kColorOnlyMesh = ResourceManager::kDefaultUnlit; MaterialHandle kPlainMesh = ResourceManager::kDefaultLit; @@ -846,6 +847,32 @@ void FilamentScene::UpdateDefaultLit(GeometryMaterialInstance& geom_mi) { .Finish(); } +void FilamentScene::UpdateGaussianSplat(GeometryMaterialInstance& geom_mi) { + auto& material = geom_mi.properties; + auto& maps = geom_mi.maps; + + renderer_.ModifyMaterial(geom_mi.mat_instance) + .SetColor("baseColor", material.base_color, false) + .SetParameter("pointSize", material.point_size) + .SetParameter("baseRoughness", material.base_roughness) + .SetParameter("baseMetallic", material.base_metallic) + .SetParameter("reflectance", material.base_reflectance) + .SetParameter("clearCoat", material.base_clearcoat) + .SetParameter("clearCoatRoughness", + material.base_clearcoat_roughness) + .SetParameter("anisotropy", material.base_anisotropy) + .SetParameter("shDegree", material.sh_degree) + .SetTexture("albedo", maps.albedo_map, + rendering::TextureSamplerParameters::Pretty()) + .SetTexture("normalMap", maps.normal_map, + rendering::TextureSamplerParameters::Pretty()) + .SetTexture("ao_rough_metalMap", maps.ao_rough_metal_map, + rendering::TextureSamplerParameters::Pretty()) + .SetTexture("reflectanceMap", maps.reflectance_map, + rendering::TextureSamplerParameters::Pretty()) + .Finish(); +} + void FilamentScene::UpdateDefaultLitSSR(GeometryMaterialInstance& geom_mi) { auto& material = geom_mi.properties; auto& maps = geom_mi.maps; @@ -1131,6 +1158,8 @@ void FilamentScene::UpdateMaterialProperties(RenderableGeometry& geom) { UpdateLineShader(geom.mat); } else if (props.shader == "unlitPolygonOffset") { UpdateUnlitPolygonOffsetShader(geom.mat); + } else if (props.shader == "gaussianSplat") { + UpdateGaussianSplat(geom.mat); } else { utility::LogWarning("'{}' is not a valid shader", props.shader); } diff --git a/cpp/open3d/visualization/rendering/filament/FilamentScene.h b/cpp/open3d/visualization/rendering/filament/FilamentScene.h index 4556f895441..f6682851d30 100644 --- a/cpp/open3d/visualization/rendering/filament/FilamentScene.h +++ b/cpp/open3d/visualization/rendering/filament/FilamentScene.h @@ -305,6 +305,7 @@ class FilamentScene : public Scene { bool shader_only = false); void UpdateMaterialProperties(RenderableGeometry& geom); void UpdateDefaultLit(GeometryMaterialInstance& geom_mi); + void UpdateGaussianSplat(GeometryMaterialInstance& geom_mi); void UpdateDefaultLitSSR(GeometryMaterialInstance& geom_mi); void UpdateDefaultUnlit(GeometryMaterialInstance& geom_mi); void UpdateNormalShader(GeometryMaterialInstance& geom_mi); diff --git a/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp b/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp new file mode 100644 index 00000000000..e5dc7b1c92d --- /dev/null +++ b/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp @@ -0,0 +1,228 @@ +// ---------------------------------------------------------------------------- +// - Open3D: www.open3d.org - +// ---------------------------------------------------------------------------- +// Copyright (c) 2018-2024 www.open3d.org +// SPDX-License-Identifier: MIT +// ---------------------------------------------------------------------------- + +// 4068: Filament has some clang-specific vectorizing pragma's that MSVC flags +// 4146: Filament's utils/algorithm.h utils::details::ctz() tries to negate +// an unsigned int. +// 4293: Filament's utils/algorithm.h utils::details::clz() does strange +// things with MSVC. Somehow sizeof(unsigned int) > 4, but its size is +// 32 so that x >> 32 gives a warning. (Or maybe the compiler can't +// determine the if statement does not run.) +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable : 4068 4146 4293) +#endif // _MSC_VER + +#include +#include +#include + +#ifdef _MSC_VER +#pragma warning(pop) +#endif // _MSC_VER + +#include "open3d/geometry/BoundingVolume.h" +#include "open3d/geometry/PointCloud.h" +#include "open3d/t/geometry/PointCloud.h" +#include "open3d/visualization/rendering/filament/FilamentEngine.h" +#include "open3d/visualization/rendering/filament/FilamentGeometryBuffersBuilder.h" +#include "open3d/visualization/rendering/filament/FilamentResourceManager.h" + +using namespace filament; + +namespace open3d { +namespace visualization { +namespace rendering { +TGaussianSplatBuffersBuilder::TGaussianSplatBuffersBuilder( + const t::geometry::PointCloud& geometry) + : TPointCloudBuffersBuilder(geometry) { + if (!geometry.IsGaussianSplat()) { + utility::LogWarning( + "TGaussianSplatBuffers is constructed for a geometry that is " + "not GaussianSplat."); + } + + std::vector check_list = {"f_dc", "opacity", "rot", "scale", + "f_rest"}; + for (const auto& check_item : check_list) { + if (geometry_.HasPointAttr(check_item) && + geometry_.GetPointAttr(check_item).GetDtype() != core::Float32) { + auto check_item_instance = geometry_.GetPointAttr(check_item); + utility::LogWarning( + "Tensor gaussian splat {} must have DType of Float32 not " + "{}. " + "Converting.", + check_item, check_item_instance.GetDtype().ToString()); + geometry_.GetPointAttr(check_item) = + check_item_instance.To(core::Float32); + } + } +} + +GeometryBuffersBuilder::Buffers +TGaussianSplatBuffersBuilder::ConstructBuffers() { + auto& engine = EngineInstance::GetInstance(); + auto& resource_mgr = EngineInstance::GetResourceManager(); + + const auto& points = geometry_.GetPointPositions(); + const size_t n_vertices = points.GetLength(); + + int sh_degree = geometry_.GaussianSplatGetSHOrder(); + if (sh_degree > 2) { + utility::LogWarning( + "Rendering for Gaussian splats with SH degrees higher than 2 " + "is not supported. They are processed as SH degree 2."); + sh_degree = 2; + } + + int f_rest_coeffs_count = sh_degree * (sh_degree + 2) * 3; + int f_rest_buffer_count = (f_rest_coeffs_count % 4 == 0) + ? (f_rest_coeffs_count / 4) + : std::ceil(f_rest_coeffs_count / 4.0); + + int base_buffer_count = 5; + int all_buffer_count = base_buffer_count + f_rest_buffer_count; + + // we use POSITION for positions, COLOR for scale, CUSTOM7 for rot + // CUSTOM0 for f_dc and opacity, CUSTOM1-CUSTOM6 for f_rest. + VertexBuffer::Builder buffer_builder = + VertexBuffer::Builder() + .bufferCount(all_buffer_count) + .vertexCount(uint32_t(n_vertices)) + .attribute(VertexAttribute::POSITION, 0, + VertexBuffer::AttributeType::FLOAT3) + .attribute(VertexAttribute::COLOR, 1, + VertexBuffer::AttributeType::FLOAT4) + .attribute(VertexAttribute::TANGENTS, 2, + VertexBuffer::AttributeType::FLOAT) + .attribute(VertexAttribute::CUSTOM0, 3, + VertexBuffer::AttributeType::FLOAT4) + .attribute(VertexAttribute::CUSTOM7, 4, + VertexBuffer::AttributeType::FLOAT4); + if (sh_degree >= 1) { + buffer_builder.attribute(VertexAttribute::CUSTOM1, 5, + VertexBuffer::AttributeType::FLOAT4); + buffer_builder.attribute(VertexAttribute::CUSTOM2, 6, + VertexBuffer::AttributeType::FLOAT4); + buffer_builder.attribute(VertexAttribute::CUSTOM3, 7, + VertexBuffer::AttributeType::FLOAT4); + } + if (sh_degree == 2) { + buffer_builder.attribute(VertexAttribute::CUSTOM4, 8, + VertexBuffer::AttributeType::FLOAT4); + buffer_builder.attribute(VertexAttribute::CUSTOM5, 9, + VertexBuffer::AttributeType::FLOAT4); + buffer_builder.attribute(VertexAttribute::CUSTOM6, 10, + VertexBuffer::AttributeType::FLOAT4); + } + + VertexBuffer* vbuf = buffer_builder.build(engine); + + VertexBufferHandle vb_handle; + if (vbuf) { + vb_handle = resource_mgr.AddVertexBuffer(vbuf); + } else { + return {}; + } + + const size_t vertex_array_size = n_vertices * 3 * sizeof(float); + float* vertex_array = static_cast(malloc(vertex_array_size)); + memcpy(vertex_array, points.GetDataPtr(), vertex_array_size); + VertexBuffer::BufferDescriptor pts_descriptor( + vertex_array, vertex_array_size, + GeometryBuffersBuilder::DeallocateBuffer); + vbuf->setBufferAt(engine, 0, std::move(pts_descriptor)); + + const size_t scale_array_size = n_vertices * 4 * sizeof(float); + float* scale_array = static_cast(malloc(scale_array_size)); + std::memset(scale_array, 0, scale_array_size); + float* scale_src = geometry_.GetPointAttr("scale").GetDataPtr(); + for (size_t i = 0; i < n_vertices; i++) { + std::memcpy(scale_array + i * 4, scale_src + i * 3, 3 * sizeof(float)); + } + VertexBuffer::BufferDescriptor scale_descriptor( + scale_array, scale_array_size, + GeometryBuffersBuilder::DeallocateBuffer); + vbuf->setBufferAt(engine, 1, std::move(scale_descriptor)); + + // We need to allocate a buffer for TANGENTS; otherwise, Filament will issue + // a warning. + const size_t empty_array_size = n_vertices * sizeof(float); + float* empty_array = static_cast(malloc(empty_array_size)); + std::memset(empty_array, 0, empty_array_size); + VertexBuffer::BufferDescriptor empty_descriptor( + empty_array, empty_array_size, + GeometryBuffersBuilder::DeallocateBuffer); + vbuf->setBufferAt(engine, 2, std::move(empty_descriptor)); + + const size_t color_array_size = n_vertices * 4 * sizeof(float); + float* color_array = static_cast(malloc(color_array_size)); + float* f_dc_ptr = geometry_.GetPointAttr("f_dc").GetDataPtr(); + float* opacity_ptr = geometry_.GetPointAttr("opacity").GetDataPtr(); + for (size_t i = 0; i < n_vertices; i++) { + std::memcpy(color_array + i * 4, f_dc_ptr + i * 3, 3 * sizeof(float)); + std::memcpy(color_array + i * 4 + 3, opacity_ptr + i, sizeof(float)); + } + VertexBuffer::BufferDescriptor color_descriptor( + color_array, color_array_size, + GeometryBuffersBuilder::DeallocateBuffer); + vbuf->setBufferAt(engine, 3, std::move(color_descriptor)); + + const size_t rot_array_size = n_vertices * 4 * sizeof(float); + float* rot_array = static_cast(malloc(rot_array_size)); + std::memcpy(rot_array, geometry_.GetPointAttr("rot").GetDataPtr(), + rot_array_size); + VertexBuffer::BufferDescriptor rot_descriptor( + rot_array, rot_array_size, + GeometryBuffersBuilder::DeallocateBuffer); + vbuf->setBufferAt(engine, 4, std::move(rot_descriptor)); + + int data_count_in_one_buffer = 4; + const size_t f_rest_array_size = + n_vertices * data_count_in_one_buffer * sizeof(float); + const size_t custom_buffer_start_index = 5; + float* f_rest_src = + (f_rest_buffer_count > 0) + ? geometry_.GetPointAttr("f_rest").GetDataPtr() + : nullptr; + for (int i = 0; i < f_rest_buffer_count; i++) { + float* f_rest_array = static_cast(malloc(f_rest_array_size)); + + size_t copy_data_size = f_rest_array_size; + if (i == f_rest_buffer_count - 1) { + int remaining_count_in_last_iter = + data_count_in_one_buffer + + (f_rest_coeffs_count - + f_rest_buffer_count * data_count_in_one_buffer); + copy_data_size = + n_vertices * remaining_count_in_last_iter * sizeof(float); + std::memset(f_rest_array, 0, f_rest_array_size); + } + + std::memcpy(f_rest_array, + f_rest_src + i * n_vertices * data_count_in_one_buffer, + copy_data_size); + VertexBuffer::BufferDescriptor f_rest_descriptor( + f_rest_array, f_rest_array_size, + GeometryBuffersBuilder::DeallocateBuffer); + vbuf->setBufferAt(engine, custom_buffer_start_index + i, + std::move(f_rest_descriptor)); + } + + auto ib_handle = CreateIndexBuffer(n_vertices); + + IndexBufferHandle downsampled_handle; + if (n_vertices >= downsample_threshold_) { + downsampled_handle = + CreateIndexBuffer(n_vertices, downsample_threshold_); + } + + return std::make_tuple(vb_handle, ib_handle, downsampled_handle); +} +} // namespace rendering +} // namespace visualization +} // namespace open3d diff --git a/cpp/open3d/visualization/rendering/filament/PointCloudBuffers.cpp b/cpp/open3d/visualization/rendering/filament/PointCloudBuffers.cpp index 2d0c1295ed6..f88db292b1f 100644 --- a/cpp/open3d/visualization/rendering/filament/PointCloudBuffers.cpp +++ b/cpp/open3d/visualization/rendering/filament/PointCloudBuffers.cpp @@ -434,15 +434,15 @@ GeometryBuffersBuilder::Buffers TPointCloudBuffersBuilder::ConstructBuffers() { const size_t uv_array_size = n_vertices * 2 * sizeof(float); float* uv_array = static_cast(malloc(uv_array_size)); if (geometry_.HasPointAttr("uv")) { - const float* uv_src = static_cast( - geometry_.GetPointAttr("uv").GetDataPtr()); + const float* uv_src = + geometry_.GetPointAttr("uv").GetDataPtr(); memcpy(uv_array, uv_src, uv_array_size); } else if (geometry_.HasPointAttr("__visualization_scalar")) { // Update in FilamentScene::UpdateGeometry(), too. memset(uv_array, 0, uv_array_size); auto vis_scalars = geometry_.GetPointAttr("__visualization_scalar").Contiguous(); - const float* src = static_cast(vis_scalars.GetDataPtr()); + const float* src = vis_scalars.GetDataPtr(); const size_t n = 2 * n_vertices; for (size_t i = 0; i < n; i += 2) { uv_array[i] = *src++; diff --git a/cpp/open3d/visualization/visualizer/O3DVisualizer.cpp b/cpp/open3d/visualization/visualizer/O3DVisualizer.cpp index 0cf3ce380cb..f5c125a1681 100644 --- a/cpp/open3d/visualization/visualizer/O3DVisualizer.cpp +++ b/cpp/open3d/visualization/visualizer/O3DVisualizer.cpp @@ -60,6 +60,7 @@ namespace visualizer { namespace { static const std::string kShaderLit = "defaultLit"; static const std::string kShaderUnlit = "defaultUnlit"; +static const std::string kShaderGaussianSplat = "gaussianSplat"; static const std::string kShaderUnlitLines = "unlitLine"; static const std::string kDefaultIBL = "default"; @@ -860,6 +861,7 @@ Ctrl-alt-click to polygon select)"; } else { // branch only applies to geometries bool has_colors = false; bool has_normals = false; + bool is_gaussian_splat = false; auto cloud = std::dynamic_pointer_cast(geom); auto lines = std::dynamic_pointer_cast(geom); @@ -886,6 +888,7 @@ Ctrl-alt-click to polygon select)"; has_colors = !cloud->colors_.empty(); has_normals = !cloud->normals_.empty(); } else if (t_cloud) { + if (t_cloud->IsGaussianSplat()) is_gaussian_splat = true; has_colors = t_cloud->HasPointColors(); has_normals = t_cloud->HasPointNormals(); } else if (lines) { @@ -932,6 +935,10 @@ Ctrl-alt-click to polygon select)"; mat.shader = kShaderLit; is_default_color = false; } + if (is_gaussian_splat) { + mat.shader = kShaderGaussianSplat; + mat.sh_degree = t_cloud->GaussianSplatGetSHOrder(); + } mat.point_size = ConvertToScaledPixels(ui_state_.point_size); // If T Geometry has a valid material convert it to MaterialRecord diff --git a/cpp/tests/Tests.h b/cpp/tests/Tests.h index f5c0cfd5b97..ba201adfea6 100644 --- a/cpp/tests/Tests.h +++ b/cpp/tests/Tests.h @@ -34,5 +34,18 @@ const Eigen::Vector2i Zero2i = Eigen::Vector2i::Zero(); // Mechanism for reporting unit tests for which there is no implementation yet. void NotImplemented(); +#define AllCloseOrShow(Arr1, Arr2, rtol, atol) \ + EXPECT_TRUE(Arr1.AllClose(Arr2, rtol, atol)) << fmt::format( \ + "Tensors are not close wrt (relative, absolute) tolerance ({}, " \ + "{}). Max error: {}\n{}\n{}", \ + rtol, atol, \ + (Arr1 - Arr2) \ + .Abs() \ + .Flatten() \ + .Max({0}) \ + .To(core::Float32) \ + .Item(), \ + Arr1.ToString(), Arr2.ToString()); + } // namespace tests } // namespace open3d diff --git a/cpp/tests/t/io/PointCloudIO.cpp b/cpp/tests/t/io/PointCloudIO.cpp index 306b617f370..54a080cac1c 100644 --- a/cpp/tests/t/io/PointCloudIO.cpp +++ b/cpp/tests/t/io/PointCloudIO.cpp @@ -217,40 +217,45 @@ TEST(TPointCloudIO, ReadPointCloudFromPLY3) { namespace { // Test ply file containig 3DGS data -const char test_3dgs_ply_data[] = R"(ply -format ascii 1.0 -element vertex 2 -property float x -property float y -property float z -property float nx -property float ny -property float nz -property float f_dc_0 -property float f_dc_1 -property float f_dc_2 -property float f_rest_0 -property float f_rest_1 -property float f_rest_2 -property float f_rest_3 -property float f_rest_4 -property float f_rest_5 -property float f_rest_6 -property float f_rest_7 -property float f_rest_8 -property float opacity -property float scale_0 -property float scale_1 -property float scale_2 -property float rot_0 -property float rot_1 -property float rot_2 -property float rot_3 -end_header -0.7236 -0.52572 -0.44721 0.48902 -0.48306 -0.7263 -1.20846 0.45058 -0.98568 -0.15648 0.03506 -0.07857 0.03506 -0.06857 0.06506 -0.03857 0.14588 -0.25489 0.56895 2.56841 0.58956 1.54784 -0.34619 -0.7938 -0.48108 0.1364 -0.6598 -1.42875 -2.85722 -0.69152 0.69793 -0.18625 0.24585 -0.3305 -1.58646 -0.15865 -0.05305 -0.12865 -0.08305 -0.11865 -0.09305 0.05648 -0.28579 -0.04457 0.33395 1.58847 2.55896 0.58984 0.55591 0.27858 -0.50835 -0.59577 -)"; - +const char test_3dgs_ply_data[] = + "ply \n" + "format ascii 1.0 \n" + "element vertex 2 \n" + "property float x \n" + "property float y \n" + "property float z \n" + "property float nx \n" + "property float ny \n" + "property float nz \n" + "property float f_dc_0 \n" + "property float f_dc_1 \n" + "property float f_dc_2 \n" + "property float f_rest_0 \n" + "property float f_rest_1 \n" + "property float f_rest_2 \n" + "property float f_rest_3 \n" + "property float f_rest_4 \n" + "property float f_rest_5 \n" + "property float f_rest_6 \n" + "property float f_rest_7 \n" + "property float f_rest_8 \n" + "property float opacity \n" + "property float scale_0 \n" + "property float scale_1 \n" + "property float scale_2 \n" + "property float rot_0 \n" + "property float rot_1 \n" + "property float rot_2 \n" + "property float rot_3 \n" + "end_header \n" + "0.7236 -0.52572 -0.44721 0.48902 -0.48306 -0.7263 -1.20846 " + "0.45058 -0.98568 -0.15648 0.03506 -0.07857 0.03506 -0.06857 " + "0.06506 -0.03857 0.14588 -0.25489 0.56895 2.56841 0.58956 1.54784 " + "-0.34619 -0.7938 -0.48108 0.1364 \n" + "0.6598 -1.42875 -2.85722 -0.69152 0.69793 -0.18625 0.24585 -0.3305 " + "-1.58646 -0.15865 -0.05305 -0.12865 -0.08305 -0.11865 -0.09305 " + "0.05648 -0.28579 -0.04457 0.33395 1.58847 2.55896 0.58984 " + "0.55591 0.27858 -0.50835 -0.59577 \n"; } // namespace // Reading ascii and check for 3DGS attributes. @@ -268,7 +273,7 @@ TEST(TPointCloudIO, ReadPointCloudFromPLY3DGS) { // Checks for scale, rot, f_dc and opacity and their shapes. EXPECT_TRUE(pcd.IsGaussianSplat()); EXPECT_TRUE(pcd.HasPointAttr("f_rest")); - // EXPECT_EQUAL(pcd.GetGaussianSplatSHOrder(), 1); // Not implemented yet. + EXPECT_EQ(pcd.GaussianSplatGetSHOrder(), 1); } // Read write empty point cloud. @@ -424,7 +429,7 @@ TEST_P(PointCloudIOPermuteDevices, ReadWrite3DGSPointCloudPLY) { t::io::ReadPointCloud(filename_3dgs_ascii, pcd_ply, {"auto", false, false, true}); EXPECT_TRUE(pcd_ply.IsGaussianSplat()); - // EXPECT_EQ(pcd_ply.GaussianSplatGetSHOrder(), 1); + EXPECT_EQ(pcd_ply.GaussianSplatGetSHOrder(), 1); auto num_gaussians_base = pcd_ply.GetPointPositions().GetLength(); EXPECT_EQ(num_gaussians_base, 2); @@ -435,18 +440,18 @@ TEST_P(PointCloudIOPermuteDevices, ReadWrite3DGSPointCloudPLY) { auto num_gaussians_new = pcd_ply_binary.GetPointPositions().GetLength(); EXPECT_EQ(num_gaussians_base, num_gaussians_new); - EXPECT_TRUE(pcd_ply.GetPointPositions().AllClose( - pcd_ply_binary.GetPointPositions())); - EXPECT_TRUE(pcd_ply.GetPointAttr("scale").AllClose( - pcd_ply_binary.GetPointAttr("scale"))); - EXPECT_TRUE(pcd_ply.GetPointAttr("opacity").AllClose( - pcd_ply_binary.GetPointAttr("opacity"))); - EXPECT_TRUE(pcd_ply.GetPointAttr("rot").AllClose( - pcd_ply_binary.GetPointAttr("rot"))); - EXPECT_TRUE(pcd_ply.GetPointAttr("f_dc").AllClose( - pcd_ply_binary.GetPointAttr("f_dc"))); - EXPECT_TRUE(pcd_ply.GetPointAttr("f_rest").AllClose( - pcd_ply_binary.GetPointAttr("f_rest"))); + AllCloseOrShow(pcd_ply.GetPointPositions(), + pcd_ply_binary.GetPointPositions(), 1e-5, 1e-8); + AllCloseOrShow(pcd_ply.GetPointAttr("scale"), + pcd_ply_binary.GetPointAttr("scale"), 1e-5, 1e-8); + AllCloseOrShow(pcd_ply.GetPointAttr("opacity"), + pcd_ply_binary.GetPointAttr("opacity"), 1e-5, 1e-8); + AllCloseOrShow(pcd_ply.GetPointAttr("rot"), + pcd_ply_binary.GetPointAttr("rot"), 1e-5, 1e-8); + AllCloseOrShow(pcd_ply.GetPointAttr("f_dc"), + pcd_ply_binary.GetPointAttr("f_dc"), 1e-5, 1e-8); + AllCloseOrShow(pcd_ply.GetPointAttr("f_rest"), + pcd_ply_binary.GetPointAttr("f_rest"), 1e-5, 1e-8); auto opacity = pcd_ply.GetPointAttr("opacity"); // Error if the shape of the attribute is not 2D with len = num_points @@ -597,19 +602,19 @@ TEST(TPointCloudIO, ReadWrite3DGSPLYToSPLAT) { t::io::ReadPointCloud(filename_splat, pcd_splat, {"auto", false, false, true}); - EXPECT_TRUE(pcd_ply.GetPointPositions().AllClose( - pcd_splat.GetPointPositions())); - EXPECT_TRUE(pcd_ply.GetPointAttr("scale").AllClose( - pcd_splat.GetPointAttr("scale"))); - EXPECT_TRUE(pcd_ply.GetPointAttr("opacity").AllClose( - pcd_splat.GetPointAttr("opacity"), 0, - 0.01)); // expect quantization errors - EXPECT_TRUE(pcd_ply.GetPointAttr("rot").AllClose( - pcd_splat.GetPointAttr("rot"), 0, - 0.01)); // expect quantization errors - EXPECT_TRUE(pcd_ply.GetPointAttr("f_dc").AllClose( - pcd_splat.GetPointAttr("f_dc"), 0, - 0.01)); // expect quantization errors + AllCloseOrShow(pcd_ply.GetPointPositions(), pcd_splat.GetPointPositions(), + 1e-5, 1e-8); + AllCloseOrShow(pcd_ply.GetPointAttr("scale"), + pcd_splat.GetPointAttr("scale"), 1e-5, 1e-8); + AllCloseOrShow(pcd_ply.GetPointAttr("opacity"), + pcd_splat.GetPointAttr("opacity"), 0, + 0.01); // expect quantization errors + AllCloseOrShow(pcd_ply.GetPointAttr("rot"), pcd_splat.GetPointAttr("rot"), + 0, + 0.01); // expect quantization errors + AllCloseOrShow(pcd_ply.GetPointAttr("f_dc"), pcd_splat.GetPointAttr("f_dc"), + 0, + 0.01); // expect quantization errors EXPECT_FALSE(pcd_splat.HasPointAttr("f_rest")); } @@ -623,20 +628,19 @@ TEST(TPointCloudIO, ReadWrite3DGSSPLAT) { std::string new_filename = utility::filesystem::GetTempDirectoryPath() + "/new_test_read.splat"; - // Write a small splat file. - // This is the same point cloud as test_3dgs_ply_data (without f_rest), - // converted to splat using the reference code from + // Write a small splat file. This is the same point cloud as + // test_3dgs_ply_data (without f_rest), converted to splat using the + // reference code from // github.com/antimatter15/splat/blob/367a9439609d043f1b23a9b455a77a977f2e7758/convert.py - // (March 2025). Converted to C array using xxd. + // (March 2025). Converted to C array with: // xxd -i test_3dgs_data.splat > test_3dgs_data_splat.h + // clang-format off const unsigned char output_splat[64] = { - 0xd9, 0x3d, 0x39, 0x3f, 0x96, 0x95, 0x06, 0xbf, 0xb6, 0xf8, 0xe4, - 0xbe, 0x96, 0xb8, 0x50, 0x41, 0x16, 0xcf, 0xe6, 0x3f, 0x16, 0x71, - 0x96, 0x40, 0x29, 0xa0, 0x39, 0xa3, 0x54, 0x1a, 0x42, 0x91, 0xa7, - 0xe8, 0x28, 0x3f, 0x48, 0xe1, 0xb6, 0xbf, 0xb1, 0xdc, 0x36, 0xc0, - 0x18, 0xae, 0x9c, 0x40, 0x08, 0xc2, 0x4e, 0x41, 0xa2, 0xdf, 0xe6, - 0x3f, 0x91, 0x68, 0x0d, 0x95, 0xc7, 0xa4, 0x3f, 0x34}; - + 0xd9, 0x3d, 0x39, 0x3f, 0x96, 0x95, 0x06, 0xbf, 0xb6, 0xf8, 0xe4, 0xbe, 0x96, 0xb8, 0x50, 0x41, + 0x16, 0xcf, 0xe6, 0x3f, 0x16, 0x71, 0x96, 0x40, 0x29, 0xa0, 0x39, 0xa3, 0x54, 0x1a, 0x42, 0x91, + 0xa7, 0xe8, 0x28, 0x3f, 0x48, 0xe1, 0xb6, 0xbf, 0xb1, 0xdc, 0x36, 0xc0, 0x18, 0xae, 0x9c, 0x40, + 0x08, 0xc2, 0x4e, 0x41, 0xa2, 0xdf, 0xe6, 0x3f, 0x91, 0x68, 0x0d, 0x95, 0xc7, 0xa4, 0x3f, 0x34}; + // clang-format on { std::ofstream outfile(filename, std::ios::binary); // Open in binary mode @@ -647,7 +651,7 @@ TEST(TPointCloudIO, ReadWrite3DGSSPLAT) { EXPECT_TRUE(t::io::ReadPointCloudFromSPLAT(filename, pcd_base, {"splat", false, false, true})); EXPECT_TRUE(pcd_base.IsGaussianSplat()); - // EXPECT_EQ(pcd_base.GaussianSplatGetSHOrder(), 0); + EXPECT_EQ(pcd_base.GaussianSplatGetSHOrder(), 0); auto num_gaussians_base = pcd_base.GetPointPositions().GetLength(); EXPECT_EQ(num_gaussians_base, 2); @@ -658,18 +662,18 @@ TEST(TPointCloudIO, ReadWrite3DGSSPLAT) { auto num_gaussians_new = pcd_new.GetPointPositions().GetLength(); EXPECT_EQ(num_gaussians_base, num_gaussians_new); - EXPECT_TRUE( - pcd_base.GetPointPositions().AllClose(pcd_new.GetPointPositions())); - EXPECT_TRUE(pcd_base.GetPointAttr("scale").AllClose( - pcd_new.GetPointAttr("scale"))); - EXPECT_TRUE(pcd_base.GetPointAttr("opacity").AllClose( - pcd_new.GetPointAttr("opacity"))); - EXPECT_TRUE(pcd_base.GetPointAttr("rot").AllClose( - pcd_new.GetPointAttr("rot"), 0, - 0.004)); // expect quantization errors - EXPECT_TRUE(pcd_base.GetPointAttr("f_dc").AllClose( - pcd_new.GetPointAttr("f_dc"), 0, - 0.004)); // expect quantization errors + AllCloseOrShow(pcd_base.GetPointPositions(), pcd_new.GetPointPositions(), + 1e-5, 1e-8); + AllCloseOrShow(pcd_base.GetPointAttr("scale"), + pcd_new.GetPointAttr("scale"), 1e-5, 1e-8); + AllCloseOrShow(pcd_base.GetPointAttr("opacity"), + pcd_new.GetPointAttr("opacity"), 0, + 0.01); // expect quantization errors + AllCloseOrShow(pcd_base.GetPointAttr("rot"), pcd_new.GetPointAttr("rot"), 0, + 0.01); // expect quantization errors + AllCloseOrShow(pcd_base.GetPointAttr("f_dc"), pcd_new.GetPointAttr("f_dc"), + 0, + 0.01); // expect quantization errors } } // namespace tests diff --git a/util/ci_utils.sh b/util/ci_utils.sh index 0d798ca9139..ba089901009 100644 --- a/util/ci_utils.sh +++ b/util/ci_utils.sh @@ -76,7 +76,7 @@ install_python_dependencies() { fi if [ "$BUILD_PYTORCH_OPS" == "ON" ]; then # ML/requirements-torch.txt if [[ "$OSTYPE" == "linux-gnu"* && "$BUILD_SYCL_MODULE" == "OFF" ]]; then - python -m pip install -U "${TORCH_GLNX}" -f "$TORCH_REPO_URL" + python -m pip install -U "${TORCH_GLNX}" -f "$TORCH_REPO_URL" python -m pip install tensorboard elif [[ "$OSTYPE" == "linux-gnu"* && "$BUILD_SYCL_MODULE" == "ON" ]]; then python -m pip install -U "${TORCH_GLNX}.cxx11.abi" -i "$TORCH_CXX11_URL" @@ -356,7 +356,9 @@ install_docs_dependencies() { command -v python python -V python -m pip install -U -q "pip==$PIP_VER" - which cmake || python -m pip install -U -q cmake + # cmake 4.0 breaks librealsense. Remove restriction when librealsense is + # updated. + which cmake || python -m pip install -U -q cmake <4.0 python -m pip install -U -q -r "${OPEN3D_SOURCE_ROOT}/python/requirements_build.txt" if [[ -d "$1" ]]; then OPEN3D_ML_ROOT="$1"