From 927463ef9687e38edb9788f05e3c2f671296f079 Mon Sep 17 00:00:00 2001 From: Zhang Yang Date: Thu, 6 Mar 2025 22:42:36 +0800 Subject: [PATCH 01/28] enable gaussian rendering based on filament --- cpp/open3d/t/geometry/PointCloud.cpp | 10 + cpp/open3d/t/geometry/PointCloud.h | 1 + cpp/open3d/visualization/CMakeLists.txt | 1 + cpp/open3d/visualization/gui/CMakeLists.txt | 1 + .../visualization/gui/Materials/gaussian.mat | 65 ++++ .../FilamentGeometryBuffersBuilder.cpp | 11 +- .../filament/FilamentGeometryBuffersBuilder.h | 14 + .../filament/FilamentResourceManager.cpp | 21 + .../filament/FilamentResourceManager.h | 1 + .../rendering/filament/FilamentScene.cpp | 6 +- .../rendering/filament/GaussianBuffers.cpp | 361 ++++++++++++++++++ .../visualizer/O3DVisualizer.cpp | 2 + 12 files changed, 490 insertions(+), 4 deletions(-) create mode 100644 cpp/open3d/visualization/gui/Materials/gaussian.mat create mode 100644 cpp/open3d/visualization/rendering/filament/GaussianBuffers.cpp diff --git a/cpp/open3d/t/geometry/PointCloud.cpp b/cpp/open3d/t/geometry/PointCloud.cpp index 9b48eea0170..19961aa6494 100644 --- a/cpp/open3d/t/geometry/PointCloud.cpp +++ b/cpp/open3d/t/geometry/PointCloud.cpp @@ -1369,6 +1369,16 @@ core::Tensor PointCloud::ComputeMetrics(const PointCloud &pcd2, metrics, params); } +bool PointCloud::IsGaussian() const { + std::vector keys_to_check = {"f_dc", "f_rest", "normals", "opacity", "rot", "scale"}; + for (const auto& key : keys_to_check) { + if (point_attr_.find(key) != point_attr_.end()) { + return true; + } + } + return false; +} + } // namespace geometry } // namespace t } // namespace open3d diff --git a/cpp/open3d/t/geometry/PointCloud.h b/cpp/open3d/t/geometry/PointCloud.h index 2dff4e6aa9f..0f01d933ae4 100644 --- a/cpp/open3d/t/geometry/PointCloud.h +++ b/cpp/open3d/t/geometry/PointCloud.h @@ -738,6 +738,7 @@ class PointCloud : public Geometry, public DrawableGeometry { std::vector metrics = {Metric::ChamferDistance}, MetricParameters params = MetricParameters()) const; + bool IsGaussian() const; protected: core::Device device_ = core::Device("CPU:0"); TensorMap point_attr_; diff --git a/cpp/open3d/visualization/CMakeLists.txt b/cpp/open3d/visualization/CMakeLists.txt index 8cf18c2088c..d1e1f3eb41e 100644 --- a/cpp/open3d/visualization/CMakeLists.txt +++ b/cpp/open3d/visualization/CMakeLists.txt @@ -80,6 +80,7 @@ if (BUILD_GUI) rendering/filament/LineSetBuffers.cpp rendering/filament/PointCloudBuffers.cpp rendering/filament/TriangleMeshBuffers.cpp + rendering/filament/GaussianBuffers.cpp ) target_sources(visualization PRIVATE diff --git a/cpp/open3d/visualization/gui/CMakeLists.txt b/cpp/open3d/visualization/gui/CMakeLists.txt index a3901dc2b0a..76280a692e3 100644 --- a/cpp/open3d/visualization/gui/CMakeLists.txt +++ b/cpp/open3d/visualization/gui/CMakeLists.txt @@ -98,6 +98,7 @@ open3d_add_compiled_materials(materials Materials/unlitLine.mat Materials/unlitPolygonOffset.mat Materials/unlitSolidColor.mat + Materials/gaussian.mat ) # Source group for Visual Studio diff --git a/cpp/open3d/visualization/gui/Materials/gaussian.mat b/cpp/open3d/visualization/gui/Materials/gaussian.mat new file mode 100644 index 00000000000..3577d35a77e --- /dev/null +++ b/cpp/open3d/visualization/gui/Materials/gaussian.mat @@ -0,0 +1,65 @@ +material { + name : defaultLit, + shadingModel : lit, + doubleSided : true, + + parameters : [ + { type : float3, name : baseColor }, + { type : float, name : baseMetallic }, + { type : float, name : baseRoughness }, + { type : float, name : reflectance }, + { type : float, name : clearCoat }, + { type : float, name : clearCoatRoughness }, + { type : float, name : anisotropy }, + { type : float, name : pointSize }, + { type : sampler2d, name : albedo }, + { type : sampler2d, name : ao_rough_metalMap }, + { type : sampler2d, name : normalMap }, + { type : sampler2d, name : reflectanceMap }, +// NOTE: Disable clear coat textures to avoid Filament warning about exceeding +// 16 sampler limit. There is currently no supported standard format that +// supports these textures so they don't get used any way. Once the shader has +// been updated to combine roughness/metallic/reflectance/ao these textures +// will be reenabled. +// +// { type : sampler2d, name : clearCoatMap }, +// { type : sampler2d, name : clearCoatRoughnessMap }, + { type : sampler2d, name : anisotropyMap } + ], + requires : [ + color, uv0 + ] +} + +vertex { + void materialVertex(inout MaterialVertexInputs material) { + gl_PointSize = materialParams.pointSize; + } +} + +fragment { + void material(inout MaterialInputs material) { + // Modifications to normal must be done before prepareMaterial call + material.normal = texture(materialParams_normalMap, getUV0()).xyz * 2.0 - 1.0; + material.normal.y *= -1.0; // NOTE: including this line because Filament samples do + // Flip normals for point clouds in case of inspection mode + // if (dot(getWorldGeometricNormalVector(), getWorldViewVector()) < 0.0) + // material.normal = -material.normal; + prepareMaterial(material); + + material.baseColor.rgb = getColor().rgb * materialParams.baseColor * texture(materialParams_albedo, getUV0()).rgb; + float3 ao_rough_metal = texture(materialParams_ao_rough_metalMap, getUV0()).rgb; + material.metallic = materialParams.baseMetallic * ao_rough_metal.b; + material.roughness = materialParams.baseRoughness * ao_rough_metal.g; + material.ambientOcclusion = ao_rough_metal.r; + //material.reflectance = materialParams.reflectance; + //material.anisotropy = materialParams.anisotropy; + material.reflectance = materialParams.reflectance * texture(materialParams_reflectanceMap, getUV0()).r; + // See note above about following disabled textures + material.clearCoat = materialParams.clearCoat; + //material.clearCoat = materialParams.clearCoat * texture(materialParams_clearCoatMap, getUV0()).r; + material.clearCoatRoughness = materialParams.clearCoatRoughness; + //material.clearCoatRoughness = materialParams.clearCoatRoughness * texture(materialParams_clearCoatRoughnessMap, getUV0()).r; + material.anisotropy = materialParams.anisotropy * texture(materialParams_anisotropyMap, getUV0()).r; + } +} diff --git a/cpp/open3d/visualization/rendering/filament/FilamentGeometryBuffersBuilder.cpp b/cpp/open3d/visualization/rendering/filament/FilamentGeometryBuffersBuilder.cpp index fe3dbd39bf7..ed47bb46a53 100644 --- a/cpp/open3d/visualization/rendering/filament/FilamentGeometryBuffersBuilder.cpp +++ b/cpp/open3d/visualization/rendering/filament/FilamentGeometryBuffersBuilder.cpp @@ -250,9 +250,14 @@ std::unique_ptr GeometryBuffersBuilder::GetBuilder( using GT = t::geometry::Geometry::GeometryType; switch (geometry.GetGeometryType()) { - case GT::PointCloud: - return std::make_unique( - static_cast(geometry)); + case GT::PointCloud: { + const t::geometry::PointCloud& pointcloud = static_cast(geometry); + if (pointcloud.IsGaussian()) { + return std::make_unique(pointcloud); + } else { + return std::make_unique(pointcloud); + } + } case GT::TriangleMesh: return std::make_unique( static_cast(geometry)); diff --git a/cpp/open3d/visualization/rendering/filament/FilamentGeometryBuffersBuilder.h b/cpp/open3d/visualization/rendering/filament/FilamentGeometryBuffersBuilder.h index afaa4a84831..5d098ce78a9 100644 --- a/cpp/open3d/visualization/rendering/filament/FilamentGeometryBuffersBuilder.h +++ b/cpp/open3d/visualization/rendering/filament/FilamentGeometryBuffersBuilder.h @@ -176,6 +176,20 @@ class TPointCloudBuffersBuilder : public GeometryBuffersBuilder { t::geometry::PointCloud geometry_; }; +class TGaussianBuffersBuilder : public GeometryBuffersBuilder { +public: + explicit TGaussianBuffersBuilder(const t::geometry::PointCloud& geometry); + + filament::RenderableManager::PrimitiveType GetPrimitiveType() + const override; + + Buffers ConstructBuffers() override; + filament::Box ComputeAABB() override; + +private: + t::geometry::PointCloud geometry_; +}; + class TLineSetBuffersBuilder : public GeometryBuffersBuilder { public: explicit TLineSetBuffersBuilder(const t::geometry::LineSet& geometry); diff --git a/cpp/open3d/visualization/rendering/filament/FilamentResourceManager.cpp b/cpp/open3d/visualization/rendering/filament/FilamentResourceManager.cpp index baee308e93c..aafc3c26345 100644 --- a/cpp/open3d/visualization/rendering/filament/FilamentResourceManager.cpp +++ b/cpp/open3d/visualization/rendering/filament/FilamentResourceManager.cpp @@ -318,6 +318,8 @@ TextureSettings GetSettingsFromImage(const t::geometry::Image& image, const MaterialHandle FilamentResourceManager::kDefaultLit = MaterialHandle::Next(); +const MaterialHandle FilamentResourceManager::kGaussian = + MaterialHandle::Next(); const MaterialHandle FilamentResourceManager::kDefaultLitWithTransparency = MaterialHandle::Next(); const MaterialHandle FilamentResourceManager::kDefaultLitSSR = @@ -1017,6 +1019,25 @@ void FilamentResourceManager::LoadDefaults() { lit_mat->setDefaultParameter("anisotropyMap", texture, default_sampler); materials_[kDefaultLit] = BoxResource(lit_mat, engine_); + const auto gaussian_path = resource_root + "/gaussian.filamat"; + auto gaussian_mat = LoadMaterialFromFile(gaussian_path, engine_); + gaussian_mat->setDefaultParameter("baseColor", filament::RgbType::sRGB, + default_color); + gaussian_mat->setDefaultParameter("baseRoughness", 0.7f); + gaussian_mat->setDefaultParameter("reflectance", 0.5f); + gaussian_mat->setDefaultParameter("baseMetallic", 0.f); + gaussian_mat->setDefaultParameter("clearCoat", 0.f); + gaussian_mat->setDefaultParameter("clearCoatRoughness", 0.f); + gaussian_mat->setDefaultParameter("anisotropy", 0.f); + gaussian_mat->setDefaultParameter("pointSize", 3.f); + gaussian_mat->setDefaultParameter("albedo", texture, default_sampler); + gaussian_mat->setDefaultParameter("ao_rough_metalMap", texture, default_sampler); + gaussian_mat->setDefaultParameter("normalMap", normal_map, default_sampler); + gaussian_mat->setDefaultParameter("reflectanceMap", texture, default_sampler); + + gaussian_mat->setDefaultParameter("anisotropyMap", texture, default_sampler); + materials_[kGaussian] = BoxResource(gaussian_mat, engine_); + const auto lit_trans_path = resource_root + "/defaultLitTransparency.filamat"; auto lit_trans_mat = LoadMaterialFromFile(lit_trans_path, engine_); diff --git a/cpp/open3d/visualization/rendering/filament/FilamentResourceManager.h b/cpp/open3d/visualization/rendering/filament/FilamentResourceManager.h index d3a6716bbab..b859f3c9066 100644 --- a/cpp/open3d/visualization/rendering/filament/FilamentResourceManager.h +++ b/cpp/open3d/visualization/rendering/filament/FilamentResourceManager.h @@ -50,6 +50,7 @@ namespace rendering { class FilamentResourceManager { public: static const MaterialHandle kDefaultLit; + static const MaterialHandle kGaussian; static const MaterialHandle kDefaultLitWithTransparency; static const MaterialHandle kDefaultLitSSR; static const MaterialHandle kDefaultUnlit; diff --git a/cpp/open3d/visualization/rendering/filament/FilamentScene.cpp b/cpp/open3d/visualization/rendering/filament/FilamentScene.cpp index ecb6c5c89c0..94944981cd6 100644 --- a/cpp/open3d/visualization/rendering/filament/FilamentScene.cpp +++ b/cpp/open3d/visualization/rendering/filament/FilamentScene.cpp @@ -102,7 +102,8 @@ std::unordered_map shader_mappings = { ResourceManager::kDefaultUnlitPolygonOffsetShader}, {"unlitBackground", ResourceManager::kDefaultUnlitBackgroundShader}, {"infiniteGroundPlane", ResourceManager::kInfinitePlaneShader}, - {"unlitLine", ResourceManager::kDefaultLineShader}}; + {"unlitLine", ResourceManager::kDefaultLineShader}, + {"gaussian", ResourceManager::kGaussian}}; MaterialHandle kColorOnlyMesh = ResourceManager::kDefaultUnlit; MaterialHandle kPlainMesh = ResourceManager::kDefaultLit; @@ -1131,6 +1132,9 @@ void FilamentScene::UpdateMaterialProperties(RenderableGeometry& geom) { UpdateLineShader(geom.mat); } else if (props.shader == "unlitPolygonOffset") { UpdateUnlitPolygonOffsetShader(geom.mat); + } else if (props.shader == "gaussian") { + UpdateDefaultLit(geom.mat); + // need to make changes here, just update the parameter } else { utility::LogWarning("'{}' is not a valid shader", props.shader); } diff --git a/cpp/open3d/visualization/rendering/filament/GaussianBuffers.cpp b/cpp/open3d/visualization/rendering/filament/GaussianBuffers.cpp new file mode 100644 index 00000000000..40fbd005e2f --- /dev/null +++ b/cpp/open3d/visualization/rendering/filament/GaussianBuffers.cpp @@ -0,0 +1,361 @@ +// ---------------------------------------------------------------------------- +// - Open3D: www.open3d.org - +// ---------------------------------------------------------------------------- +// Copyright (c) 2018-2024 www.open3d.org +// SPDX-License-Identifier: MIT +// ---------------------------------------------------------------------------- + +// 4068: Filament has some clang-specific vectorizing pragma's that MSVC flags +// 4146: Filament's utils/algorithm.h utils::details::ctz() tries to negate +// an unsigned int. +// 4293: Filament's utils/algorithm.h utils::details::clz() does strange +// things with MSVC. Somehow sizeof(unsigned int) > 4, but its size is +// 32 so that x >> 32 gives a warning. (Or maybe the compiler can't +// determine the if statement does not run.) +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable : 4068 4146 4293) +#endif // _MSC_VER + +#include +#include +#include + +#ifdef _MSC_VER +#pragma warning(pop) +#endif // _MSC_VER + +#include "open3d/geometry/BoundingVolume.h" +#include "open3d/geometry/PointCloud.h" +#include "open3d/t/geometry/PointCloud.h" +#include "open3d/visualization/rendering/filament/FilamentEngine.h" +#include "open3d/visualization/rendering/filament/FilamentGeometryBuffersBuilder.h" +#include "open3d/visualization/rendering/filament/FilamentResourceManager.h" + +using namespace filament; + +namespace open3d { +namespace visualization { +namespace rendering { + +namespace { +struct ColoredVertex { + math::float3 position = {0.f, 0.f, 0.f}; + // Default to mid-gray which provides good separation from the two most + // common background colors: white and black. Otherwise, point clouds + // without per-vertex colors may appear is if they are not rendering because + // they blend with the background. + math::float4 color = {0.5f, 0.5f, 0.5f, 1.f}; + math::quatf tangent = {0.f, 0.f, 0.f, 1.f}; + math::float2 uv = {0.f, 0.f}; + + static std::uint32_t GetPositionOffset() { + return offsetof(ColoredVertex, position); + } + static std::uint32_t GetColorOffset() { + return offsetof(ColoredVertex, color); + } + static std::uint32_t GetTangentOffset() { + return offsetof(ColoredVertex, tangent); + } + static std::uint32_t GetUVOffset() { return offsetof(ColoredVertex, uv); } + void SetVertexPosition(const Eigen::Vector3d& pos) { + auto float_pos = pos.cast(); + position.x = float_pos(0); + position.y = float_pos(1); + position.z = float_pos(2); + } + + float sRGBToLinear(float color) { + return color <= 0.04045f ? color / 12.92f + : pow((color + 0.055f) / 1.055f, 2.4f); + } + + void SetVertexColor(const Eigen::Vector3d& c, bool adjust_for_srgb) { + auto float_color = c.cast(); + if (adjust_for_srgb) { + color.x = sRGBToLinear(float_color(0)); + color.y = sRGBToLinear(float_color(1)); + color.z = sRGBToLinear(float_color(2)); + } else { + color.x = float_color(0); + color.y = float_color(1); + color.z = float_color(2); + } + } +}; +} // namespace + +TGaussianBuffersBuilder::TGaussianBuffersBuilder( + const t::geometry::PointCloud& geometry) + : geometry_(geometry) { + // Make sure geometry is on CPU + auto pts = geometry.GetPointPositions(); + if (pts.IsCUDA()) { + utility::LogWarning( + "GPU resident point clouds are not currently supported for " + "visualization. Copying data to CPU."); + geometry_ = geometry.To(core::Device("CPU:0")); + } + + // Now make sure data types are Float32 + if (pts.GetDtype() != core::Float32) { + utility::LogWarning( + "Tensor point cloud points must have DType of Float32 not {}. " + "Converting.", + pts.GetDtype().ToString()); + geometry_.GetPointPositions() = pts.To(core::Float32); + } + if (geometry_.HasPointNormals() && + geometry_.GetPointNormals().GetDtype() != core::Float32) { + auto normals = geometry_.GetPointNormals(); + utility::LogWarning( + "Tensor point cloud normals must have DType of Float32 not {}. " + "Converting.", + normals.GetDtype().ToString()); + geometry_.GetPointNormals() = normals.To(core::Float32); + } + if (geometry_.HasPointColors() && + geometry_.GetPointColors().GetDtype() != core::Float32) { + auto colors = geometry_.GetPointColors(); + utility::LogWarning( + "Tensor point cloud colors must have DType of Float32 not {}. " + "Converting.", + colors.GetDtype().ToString()); + geometry_.GetPointColors() = colors.To(core::Float32); + if (colors.GetDtype() == core::UInt8) { + geometry_.GetPointColors() = geometry_.GetPointColors() / 255.0f; + } + } +} + +RenderableManager::PrimitiveType TGaussianBuffersBuilder::GetPrimitiveType() + const { + return RenderableManager::PrimitiveType::POINTS; +} + +GeometryBuffersBuilder::Buffers TGaussianBuffersBuilder::ConstructBuffers() { + auto& engine = EngineInstance::GetInstance(); + auto& resource_mgr = EngineInstance::GetResourceManager(); + + const auto& points = geometry_.GetPointPositions(); + const size_t n_vertices = points.GetLength(); + + // we use TANGENTS for normals, CUSTOM0 to load rot, CUSTOM1 to load scale, CUSTOM2 to load opacity, + // CUSTOM3 for f_dc, + VertexBuffer* vbuf = VertexBuffer::Builder() + .bufferCount(8) + .vertexCount(uint32_t(n_vertices)) + .attribute(VertexAttribute::POSITION, 0, + VertexBuffer::AttributeType::FLOAT3) + .normalized(VertexAttribute::COLOR) + .attribute(VertexAttribute::COLOR, 1, + VertexBuffer::AttributeType::FLOAT3) + .normalized(VertexAttribute::TANGENTS) + .attribute(VertexAttribute::TANGENTS, 2, + VertexBuffer::AttributeType::FLOAT4) + .attribute(VertexAttribute::CUSTOM0, 3, + VertexBuffer::AttributeType::FLOAT4) + .attribute(VertexAttribute::CUSTOM1, 4, + VertexBuffer::AttributeType::FLOAT3) + .attribute(VertexAttribute::CUSTOM2, 5, + VertexBuffer::AttributeType::FLOAT) + .attribute(VertexAttribute::CUSTOM3, 6, + VertexBuffer::AttributeType::FLOAT3) + // for now, we have not finish the defination of gaussian.mat file, + // so we allocate buffer for uv0 to avoid warning. + .attribute(VertexAttribute::UV0, 7, + VertexBuffer::AttributeType::FLOAT2) + .build(engine); + + VertexBufferHandle vb_handle; + if (vbuf) { + vb_handle = resource_mgr.AddVertexBuffer(vbuf); + } else { + return {}; + } + + const size_t vertex_array_size = n_vertices * 3 * sizeof(float); + float* vertex_array = static_cast(malloc(vertex_array_size)); + memcpy(vertex_array, points.GetDataPtr(), vertex_array_size); + VertexBuffer::BufferDescriptor pts_descriptor( + vertex_array, vertex_array_size, + GeometryBuffersBuilder::DeallocateBuffer); + vbuf->setBufferAt(engine, 0, std::move(pts_descriptor)); + + const size_t color_array_size = n_vertices * 3 * sizeof(float); + // for now, we save f_dc in color buffer + if (geometry_.HasPointAttr("f_dc")) { + float* color_array = static_cast(malloc(color_array_size)); + memcpy(color_array, geometry_.GetPointAttr("f_dc").GetDataPtr(), + color_array_size); + VertexBuffer::BufferDescriptor color_descriptor( + color_array, color_array_size, + GeometryBuffersBuilder::DeallocateBuffer); + vbuf->setBufferAt(engine, 1, std::move(color_descriptor)); + } else { + float* color_array = static_cast(malloc(color_array_size)); + for (size_t i = 0; i < n_vertices * 3; ++i) { + color_array[i] = 1.f; + } + VertexBuffer::BufferDescriptor color_descriptor( + color_array, color_array_size, + GeometryBuffersBuilder::DeallocateBuffer); + vbuf->setBufferAt(engine, 1, std::move(color_descriptor)); + } + + const size_t normal_array_size = n_vertices * 4 * sizeof(float); + if (geometry_.HasPointNormals()) { + const auto& normals = geometry_.GetPointNormals(); + + // Converting normals to Filament type - quaternions + auto float4v_tangents = + static_cast(malloc(normal_array_size)); + auto orientation = + filament::geometry::SurfaceOrientation::Builder() + .vertexCount(n_vertices) + .normals(reinterpret_cast( + normals.GetDataPtr())) + .build(); + orientation->getQuats(float4v_tangents, n_vertices); + VertexBuffer::BufferDescriptor normals_descriptor( + float4v_tangents, normal_array_size, + GeometryBuffersBuilder::DeallocateBuffer); + vbuf->setBufferAt(engine, 2, std::move(normals_descriptor)); + delete orientation; + } else { + float* normal_array = static_cast(malloc(normal_array_size)); + float* normal_ptr = normal_array; + for (size_t i = 0; i < n_vertices; ++i) { + *normal_ptr++ = 0.f; + *normal_ptr++ = 0.f; + *normal_ptr++ = 0.f; + *normal_ptr++ = 1.f; + } + VertexBuffer::BufferDescriptor normals_descriptor( + normal_array, normal_array_size, + GeometryBuffersBuilder::DeallocateBuffer); + vbuf->setBufferAt(engine, 2, std::move(normals_descriptor)); + } + + const size_t rot_array_size = n_vertices * 4 * sizeof(float); + if (geometry_.HasPointAttr("rot")) { + float* rot_array = static_cast(malloc(rot_array_size)); + memcpy(rot_array, geometry_.GetPointAttr("rot").GetDataPtr(), + rot_array_size); + VertexBuffer::BufferDescriptor rot_descriptor( + rot_array, rot_array_size, + GeometryBuffersBuilder::DeallocateBuffer); + vbuf->setBufferAt(engine, 3, std::move(rot_descriptor)); + } else { + float* rot_array = static_cast(malloc(rot_array_size)); + for (size_t i = 0; i < n_vertices * 4; ++i) { + rot_array[i] = 1.f; + } + VertexBuffer::BufferDescriptor rot_descriptor( + rot_array, rot_array_size, + GeometryBuffersBuilder::DeallocateBuffer); + vbuf->setBufferAt(engine, 3, std::move(rot_descriptor)); + } + + const size_t scale_array_size = n_vertices * 3 * sizeof(float); + float* scale_array = static_cast(malloc(scale_array_size)); + if (geometry_.HasPointAttr("scale")) { + const float* scale_src = static_cast( + geometry_.GetPointAttr("scale").GetDataPtr()); + memcpy(scale_array, scale_src, scale_array_size); + } else { + memset(scale_array, 0, scale_array_size); + } + VertexBuffer::BufferDescriptor scale_descriptor( + scale_array, scale_array_size, GeometryBuffersBuilder::DeallocateBuffer); + vbuf->setBufferAt(engine, 4, std::move(scale_descriptor)); + + const size_t opacity_array_size = n_vertices * 1 * sizeof(float); + float* opacity_array = static_cast(malloc(opacity_array_size)); + if (geometry_.HasPointAttr("opacity")) { + const float* opacity_src = static_cast( + geometry_.GetPointAttr("opacity").GetDataPtr()); + memcpy(opacity_array, opacity_src, opacity_array_size); + } else { + memset(opacity_array, 0, opacity_array_size); + } + VertexBuffer::BufferDescriptor opacity_descriptor( + opacity_array, opacity_array_size, GeometryBuffersBuilder::DeallocateBuffer); + vbuf->setBufferAt(engine, 5, std::move(opacity_descriptor)); + + const size_t f_dc_array_size = n_vertices * 3 * sizeof(float); + float* f_dc_array = static_cast(malloc(f_dc_array_size)); + if (geometry_.HasPointAttr("f_dc")) { + const float* f_dc_src = static_cast( + geometry_.GetPointAttr("f_dc").GetDataPtr()); + memcpy(f_dc_array, f_dc_src, f_dc_array_size); + } else { + memset(f_dc_array, 0, f_dc_array_size); + } + VertexBuffer::BufferDescriptor f_dc_descriptor( + f_dc_array, f_dc_array_size, GeometryBuffersBuilder::DeallocateBuffer); + vbuf->setBufferAt(engine, 6, std::move(f_dc_descriptor)); + + const size_t uv_array_size = n_vertices * 2 * sizeof(float); + float* uv_array = static_cast(malloc(uv_array_size)); + if (geometry_.HasPointAttr("uv")) { + const float* uv_src = static_cast( + geometry_.GetPointAttr("uv").GetDataPtr()); + memcpy(uv_array, uv_src, uv_array_size); + } else if (geometry_.HasPointAttr("__visualization_scalar")) { + // Update in FilamentScene::UpdateGeometry(), too. + memset(uv_array, 0, uv_array_size); + auto vis_scalars = + geometry_.GetPointAttr("__visualization_scalar").Contiguous(); + const float* src = static_cast(vis_scalars.GetDataPtr()); + const size_t n = 2 * n_vertices; + for (size_t i = 0; i < n; i += 2) { + uv_array[i] = *src++; + } + } else { + memset(uv_array, 0, uv_array_size); + } + VertexBuffer::BufferDescriptor uv_descriptor( + uv_array, uv_array_size, GeometryBuffersBuilder::DeallocateBuffer); + vbuf->setBufferAt(engine, 7, std::move(uv_descriptor)); + + auto ib_handle = CreateIndexBuffer(n_vertices); + + IndexBufferHandle downsampled_handle; + if (n_vertices >= downsample_threshold_) { + downsampled_handle = + CreateIndexBuffer(n_vertices, downsample_threshold_); + } + + return std::make_tuple(vb_handle, ib_handle, downsampled_handle); +} + +filament::Box TGaussianBuffersBuilder::ComputeAABB() { + auto min_bounds = geometry_.GetMinBound(); + auto max_bounds = geometry_.GetMaxBound(); + auto* min_bounds_float = min_bounds.GetDataPtr(); + auto* max_bounds_float = max_bounds.GetDataPtr(); + + filament::math::float3 min(min_bounds_float[0], min_bounds_float[1], + min_bounds_float[2]); + filament::math::float3 max(max_bounds_float[0], max_bounds_float[1], + max_bounds_float[2]); + + Box aabb; + aabb.set(min, max); + if (aabb.isEmpty()) { + min.x -= 0.1f; + min.y -= 0.1f; + min.z -= 0.1f; + max.x += 0.1f; + max.y += 0.1f; + max.z += 0.1f; + aabb.set(min, max); + } + return aabb; +} + +} // namespace rendering +} // namespace visualization +} // namespace open3d diff --git a/cpp/open3d/visualization/visualizer/O3DVisualizer.cpp b/cpp/open3d/visualization/visualizer/O3DVisualizer.cpp index 0cf3ce380cb..77f12108e6e 100644 --- a/cpp/open3d/visualization/visualizer/O3DVisualizer.cpp +++ b/cpp/open3d/visualization/visualizer/O3DVisualizer.cpp @@ -60,6 +60,7 @@ namespace visualizer { namespace { static const std::string kShaderLit = "defaultLit"; static const std::string kShaderUnlit = "defaultUnlit"; +static const std::string kGaussian = "gaussian"; static const std::string kShaderUnlitLines = "unlitLine"; static const std::string kDefaultIBL = "default"; @@ -930,6 +931,7 @@ Ctrl-alt-click to polygon select)"; if (has_normals) { mat.base_color = {1.0f, 1.0f, 1.0f, 1.0f}; mat.shader = kShaderLit; + mat.shader = kGaussian; is_default_color = false; } mat.point_size = ConvertToScaledPixels(ui_state_.point_size); From 7a5cab69deaf9351a9d3f1d29687e624f4f5743f Mon Sep 17 00:00:00 2001 From: Zhang Yang Date: Fri, 7 Mar 2025 12:18:59 +0800 Subject: [PATCH 02/28] rename gaussian to gaussianSplat --- cpp/open3d/t/geometry/PointCloud.cpp | 10 +++++----- cpp/open3d/t/geometry/PointCloud.h | 2 +- cpp/open3d/visualization/CMakeLists.txt | 2 +- .../filament/FilamentGeometryBuffersBuilder.cpp | 4 ++-- .../filament/FilamentGeometryBuffersBuilder.h | 4 ++-- .../rendering/filament/FilamentResourceManager.cpp | 4 ++-- .../rendering/filament/FilamentResourceManager.h | 2 +- .../visualization/rendering/filament/FilamentScene.cpp | 4 ++-- .../{GaussianBuffers.cpp => GaussianSplatBuffers.cpp} | 8 ++++---- cpp/open3d/visualization/visualizer/O3DVisualizer.cpp | 4 ++-- 10 files changed, 22 insertions(+), 22 deletions(-) rename cpp/open3d/visualization/rendering/filament/{GaussianBuffers.cpp => GaussianSplatBuffers.cpp} (98%) diff --git a/cpp/open3d/t/geometry/PointCloud.cpp b/cpp/open3d/t/geometry/PointCloud.cpp index 19961aa6494..d296570fe77 100644 --- a/cpp/open3d/t/geometry/PointCloud.cpp +++ b/cpp/open3d/t/geometry/PointCloud.cpp @@ -1369,14 +1369,14 @@ core::Tensor PointCloud::ComputeMetrics(const PointCloud &pcd2, metrics, params); } -bool PointCloud::IsGaussian() const { - std::vector keys_to_check = {"f_dc", "f_rest", "normals", "opacity", "rot", "scale"}; +bool PointCloud::IsGaussianSplat() const { + std::vector keys_to_check = {"f_dc", "opacity", "rot", "scale"}; for (const auto& key : keys_to_check) { - if (point_attr_.find(key) != point_attr_.end()) { - return true; + if (point_attr_.find(key) == point_attr_.end()) { + return false; } } - return false; + return true; } } // namespace geometry diff --git a/cpp/open3d/t/geometry/PointCloud.h b/cpp/open3d/t/geometry/PointCloud.h index 0f01d933ae4..e3dd1fbf391 100644 --- a/cpp/open3d/t/geometry/PointCloud.h +++ b/cpp/open3d/t/geometry/PointCloud.h @@ -738,7 +738,7 @@ class PointCloud : public Geometry, public DrawableGeometry { std::vector metrics = {Metric::ChamferDistance}, MetricParameters params = MetricParameters()) const; - bool IsGaussian() const; + bool IsGaussianSplat() const; protected: core::Device device_ = core::Device("CPU:0"); TensorMap point_attr_; diff --git a/cpp/open3d/visualization/CMakeLists.txt b/cpp/open3d/visualization/CMakeLists.txt index d1e1f3eb41e..833ca2ae40f 100644 --- a/cpp/open3d/visualization/CMakeLists.txt +++ b/cpp/open3d/visualization/CMakeLists.txt @@ -80,7 +80,7 @@ if (BUILD_GUI) rendering/filament/LineSetBuffers.cpp rendering/filament/PointCloudBuffers.cpp rendering/filament/TriangleMeshBuffers.cpp - rendering/filament/GaussianBuffers.cpp + rendering/filament/GaussianSplatBuffers.cpp ) target_sources(visualization PRIVATE diff --git a/cpp/open3d/visualization/rendering/filament/FilamentGeometryBuffersBuilder.cpp b/cpp/open3d/visualization/rendering/filament/FilamentGeometryBuffersBuilder.cpp index ed47bb46a53..aa5d3dfa212 100644 --- a/cpp/open3d/visualization/rendering/filament/FilamentGeometryBuffersBuilder.cpp +++ b/cpp/open3d/visualization/rendering/filament/FilamentGeometryBuffersBuilder.cpp @@ -252,8 +252,8 @@ std::unique_ptr GeometryBuffersBuilder::GetBuilder( switch (geometry.GetGeometryType()) { case GT::PointCloud: { const t::geometry::PointCloud& pointcloud = static_cast(geometry); - if (pointcloud.IsGaussian()) { - return std::make_unique(pointcloud); + if (pointcloud.IsGaussianSplat()) { + return std::make_unique(pointcloud); } else { return std::make_unique(pointcloud); } diff --git a/cpp/open3d/visualization/rendering/filament/FilamentGeometryBuffersBuilder.h b/cpp/open3d/visualization/rendering/filament/FilamentGeometryBuffersBuilder.h index 5d098ce78a9..9fde9821420 100644 --- a/cpp/open3d/visualization/rendering/filament/FilamentGeometryBuffersBuilder.h +++ b/cpp/open3d/visualization/rendering/filament/FilamentGeometryBuffersBuilder.h @@ -176,9 +176,9 @@ class TPointCloudBuffersBuilder : public GeometryBuffersBuilder { t::geometry::PointCloud geometry_; }; -class TGaussianBuffersBuilder : public GeometryBuffersBuilder { +class TGaussianSplatBuffersBuilder : public GeometryBuffersBuilder { public: - explicit TGaussianBuffersBuilder(const t::geometry::PointCloud& geometry); + explicit TGaussianSplatBuffersBuilder(const t::geometry::PointCloud& geometry); filament::RenderableManager::PrimitiveType GetPrimitiveType() const override; diff --git a/cpp/open3d/visualization/rendering/filament/FilamentResourceManager.cpp b/cpp/open3d/visualization/rendering/filament/FilamentResourceManager.cpp index aafc3c26345..a9248975caf 100644 --- a/cpp/open3d/visualization/rendering/filament/FilamentResourceManager.cpp +++ b/cpp/open3d/visualization/rendering/filament/FilamentResourceManager.cpp @@ -318,7 +318,7 @@ TextureSettings GetSettingsFromImage(const t::geometry::Image& image, const MaterialHandle FilamentResourceManager::kDefaultLit = MaterialHandle::Next(); -const MaterialHandle FilamentResourceManager::kGaussian = +const MaterialHandle FilamentResourceManager::kGaussianSplatShader = MaterialHandle::Next(); const MaterialHandle FilamentResourceManager::kDefaultLitWithTransparency = MaterialHandle::Next(); @@ -1036,7 +1036,7 @@ void FilamentResourceManager::LoadDefaults() { gaussian_mat->setDefaultParameter("reflectanceMap", texture, default_sampler); gaussian_mat->setDefaultParameter("anisotropyMap", texture, default_sampler); - materials_[kGaussian] = BoxResource(gaussian_mat, engine_); + materials_[kGaussianSplatShader] = BoxResource(gaussian_mat, engine_); const auto lit_trans_path = resource_root + "/defaultLitTransparency.filamat"; diff --git a/cpp/open3d/visualization/rendering/filament/FilamentResourceManager.h b/cpp/open3d/visualization/rendering/filament/FilamentResourceManager.h index b859f3c9066..5503306b8ae 100644 --- a/cpp/open3d/visualization/rendering/filament/FilamentResourceManager.h +++ b/cpp/open3d/visualization/rendering/filament/FilamentResourceManager.h @@ -50,7 +50,7 @@ namespace rendering { class FilamentResourceManager { public: static const MaterialHandle kDefaultLit; - static const MaterialHandle kGaussian; + static const MaterialHandle kGaussianSplatShader; static const MaterialHandle kDefaultLitWithTransparency; static const MaterialHandle kDefaultLitSSR; static const MaterialHandle kDefaultUnlit; diff --git a/cpp/open3d/visualization/rendering/filament/FilamentScene.cpp b/cpp/open3d/visualization/rendering/filament/FilamentScene.cpp index 94944981cd6..9e364f91f1f 100644 --- a/cpp/open3d/visualization/rendering/filament/FilamentScene.cpp +++ b/cpp/open3d/visualization/rendering/filament/FilamentScene.cpp @@ -103,7 +103,7 @@ std::unordered_map shader_mappings = { {"unlitBackground", ResourceManager::kDefaultUnlitBackgroundShader}, {"infiniteGroundPlane", ResourceManager::kInfinitePlaneShader}, {"unlitLine", ResourceManager::kDefaultLineShader}, - {"gaussian", ResourceManager::kGaussian}}; + {"gaussianSplat", ResourceManager::kGaussianSplatShader}}; MaterialHandle kColorOnlyMesh = ResourceManager::kDefaultUnlit; MaterialHandle kPlainMesh = ResourceManager::kDefaultLit; @@ -1132,7 +1132,7 @@ void FilamentScene::UpdateMaterialProperties(RenderableGeometry& geom) { UpdateLineShader(geom.mat); } else if (props.shader == "unlitPolygonOffset") { UpdateUnlitPolygonOffsetShader(geom.mat); - } else if (props.shader == "gaussian") { + } else if (props.shader == "gaussianSplat") { UpdateDefaultLit(geom.mat); // need to make changes here, just update the parameter } else { diff --git a/cpp/open3d/visualization/rendering/filament/GaussianBuffers.cpp b/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp similarity index 98% rename from cpp/open3d/visualization/rendering/filament/GaussianBuffers.cpp rename to cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp index 40fbd005e2f..763a0767e01 100644 --- a/cpp/open3d/visualization/rendering/filament/GaussianBuffers.cpp +++ b/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp @@ -86,7 +86,7 @@ struct ColoredVertex { }; } // namespace -TGaussianBuffersBuilder::TGaussianBuffersBuilder( +TGaussianSplatBuffersBuilder::TGaussianSplatBuffersBuilder( const t::geometry::PointCloud& geometry) : geometry_(geometry) { // Make sure geometry is on CPU @@ -129,12 +129,12 @@ TGaussianBuffersBuilder::TGaussianBuffersBuilder( } } -RenderableManager::PrimitiveType TGaussianBuffersBuilder::GetPrimitiveType() +RenderableManager::PrimitiveType TGaussianSplatBuffersBuilder::GetPrimitiveType() const { return RenderableManager::PrimitiveType::POINTS; } -GeometryBuffersBuilder::Buffers TGaussianBuffersBuilder::ConstructBuffers() { +GeometryBuffersBuilder::Buffers TGaussianSplatBuffersBuilder::ConstructBuffers() { auto& engine = EngineInstance::GetInstance(); auto& resource_mgr = EngineInstance::GetResourceManager(); @@ -331,7 +331,7 @@ GeometryBuffersBuilder::Buffers TGaussianBuffersBuilder::ConstructBuffers() { return std::make_tuple(vb_handle, ib_handle, downsampled_handle); } -filament::Box TGaussianBuffersBuilder::ComputeAABB() { +filament::Box TGaussianSplatBuffersBuilder::ComputeAABB() { auto min_bounds = geometry_.GetMinBound(); auto max_bounds = geometry_.GetMaxBound(); auto* min_bounds_float = min_bounds.GetDataPtr(); diff --git a/cpp/open3d/visualization/visualizer/O3DVisualizer.cpp b/cpp/open3d/visualization/visualizer/O3DVisualizer.cpp index 77f12108e6e..37fb415e1f4 100644 --- a/cpp/open3d/visualization/visualizer/O3DVisualizer.cpp +++ b/cpp/open3d/visualization/visualizer/O3DVisualizer.cpp @@ -60,7 +60,7 @@ namespace visualizer { namespace { static const std::string kShaderLit = "defaultLit"; static const std::string kShaderUnlit = "defaultUnlit"; -static const std::string kGaussian = "gaussian"; +static const std::string kShaderGaussianSplat = "gaussianSplat"; static const std::string kShaderUnlitLines = "unlitLine"; static const std::string kDefaultIBL = "default"; @@ -931,7 +931,7 @@ Ctrl-alt-click to polygon select)"; if (has_normals) { mat.base_color = {1.0f, 1.0f, 1.0f, 1.0f}; mat.shader = kShaderLit; - mat.shader = kGaussian; + mat.shader = kShaderGaussianSplat; is_default_color = false; } mat.point_size = ConvertToScaledPixels(ui_state_.point_size); From 141a30b4457da037cb8c7434efe852f570f130c6 Mon Sep 17 00:00:00 2001 From: Zhang Yang Date: Sat, 8 Mar 2025 11:05:34 +0800 Subject: [PATCH 03/28] update buffer allocation --- .../filament/GaussianSplatBuffers.cpp | 178 +++++------------- 1 file changed, 52 insertions(+), 126 deletions(-) diff --git a/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp b/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp index 763a0767e01..a85a900a3b2 100644 --- a/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp +++ b/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp @@ -141,31 +141,31 @@ GeometryBuffersBuilder::Buffers TGaussianSplatBuffersBuilder::ConstructBuffers() const auto& points = geometry_.GetPointPositions(); const size_t n_vertices = points.GetLength(); - // we use TANGENTS for normals, CUSTOM0 to load rot, CUSTOM1 to load scale, CUSTOM2 to load opacity, - // CUSTOM3 for f_dc, + // we usePOSITION for positions, COLOR for f_dc and opacity, TANGENTS for rot + // CUSTOM0 for scale, CUSTOM1-CUSTOM6 for f_rest. VertexBuffer* vbuf = VertexBuffer::Builder() - .bufferCount(8) + .bufferCount(10) .vertexCount(uint32_t(n_vertices)) .attribute(VertexAttribute::POSITION, 0, VertexBuffer::AttributeType::FLOAT3) - .normalized(VertexAttribute::COLOR) .attribute(VertexAttribute::COLOR, 1, - VertexBuffer::AttributeType::FLOAT3) - .normalized(VertexAttribute::TANGENTS) + VertexBuffer::AttributeType::FLOAT4) .attribute(VertexAttribute::TANGENTS, 2, VertexBuffer::AttributeType::FLOAT4) .attribute(VertexAttribute::CUSTOM0, 3, - VertexBuffer::AttributeType::FLOAT4) + VertexBuffer::AttributeType::FLOAT4) .attribute(VertexAttribute::CUSTOM1, 4, - VertexBuffer::AttributeType::FLOAT3) + VertexBuffer::AttributeType::FLOAT4) .attribute(VertexAttribute::CUSTOM2, 5, - VertexBuffer::AttributeType::FLOAT) + VertexBuffer::AttributeType::FLOAT4) .attribute(VertexAttribute::CUSTOM3, 6, - VertexBuffer::AttributeType::FLOAT3) - // for now, we have not finish the defination of gaussian.mat file, - // so we allocate buffer for uv0 to avoid warning. - .attribute(VertexAttribute::UV0, 7, - VertexBuffer::AttributeType::FLOAT2) + VertexBuffer::AttributeType::FLOAT4) + .attribute(VertexAttribute::CUSTOM4, 7, + VertexBuffer::AttributeType::FLOAT4) + .attribute(VertexAttribute::CUSTOM5, 8, + VertexBuffer::AttributeType::FLOAT4) + .attribute(VertexAttribute::CUSTOM6, 9, + VertexBuffer::AttributeType::FLOAT4) .build(engine); VertexBufferHandle vb_handle; @@ -183,142 +183,68 @@ GeometryBuffersBuilder::Buffers TGaussianSplatBuffersBuilder::ConstructBuffers() GeometryBuffersBuilder::DeallocateBuffer); vbuf->setBufferAt(engine, 0, std::move(pts_descriptor)); - const size_t color_array_size = n_vertices * 3 * sizeof(float); - // for now, we save f_dc in color buffer - if (geometry_.HasPointAttr("f_dc")) { - float* color_array = static_cast(malloc(color_array_size)); - memcpy(color_array, geometry_.GetPointAttr("f_dc").GetDataPtr(), - color_array_size); - VertexBuffer::BufferDescriptor color_descriptor( - color_array, color_array_size, - GeometryBuffersBuilder::DeallocateBuffer); - vbuf->setBufferAt(engine, 1, std::move(color_descriptor)); + const size_t color_array_size = n_vertices * 4 * sizeof(float); + float* color_array = static_cast(malloc(color_array_size)); + if (geometry_.HasPointAttr("f_dc") && geometry_.HasPointAttr("opacity")) { + float* f_dc_ptr = static_cast(geometry_.GetPointAttr("f_dc").GetDataPtr()); + float* opacity_ptr = static_cast(geometry_.GetPointAttr("opacity").GetDataPtr()); + for (size_t i = 0; i < n_vertices; i++) { + std::memcpy(color_array + i * 4, f_dc_ptr + i * 3, 3 * sizeof(float)); + std::memcpy(color_array + i * 4 + 3, opacity_ptr + i, sizeof(float)); + } } else { - float* color_array = static_cast(malloc(color_array_size)); - for (size_t i = 0; i < n_vertices * 3; ++i) { + for (size_t i = 0; i < n_vertices * 4; i++) { color_array[i] = 1.f; } - VertexBuffer::BufferDescriptor color_descriptor( - color_array, color_array_size, - GeometryBuffersBuilder::DeallocateBuffer); - vbuf->setBufferAt(engine, 1, std::move(color_descriptor)); } - - const size_t normal_array_size = n_vertices * 4 * sizeof(float); - if (geometry_.HasPointNormals()) { - const auto& normals = geometry_.GetPointNormals(); - - // Converting normals to Filament type - quaternions - auto float4v_tangents = - static_cast(malloc(normal_array_size)); - auto orientation = - filament::geometry::SurfaceOrientation::Builder() - .vertexCount(n_vertices) - .normals(reinterpret_cast( - normals.GetDataPtr())) - .build(); - orientation->getQuats(float4v_tangents, n_vertices); - VertexBuffer::BufferDescriptor normals_descriptor( - float4v_tangents, normal_array_size, - GeometryBuffersBuilder::DeallocateBuffer); - vbuf->setBufferAt(engine, 2, std::move(normals_descriptor)); - delete orientation; - } else { - float* normal_array = static_cast(malloc(normal_array_size)); - float* normal_ptr = normal_array; - for (size_t i = 0; i < n_vertices; ++i) { - *normal_ptr++ = 0.f; - *normal_ptr++ = 0.f; - *normal_ptr++ = 0.f; - *normal_ptr++ = 1.f; - } - VertexBuffer::BufferDescriptor normals_descriptor( - normal_array, normal_array_size, + VertexBuffer::BufferDescriptor color_descriptor( + color_array, color_array_size, GeometryBuffersBuilder::DeallocateBuffer); - vbuf->setBufferAt(engine, 2, std::move(normals_descriptor)); - } + vbuf->setBufferAt(engine, 1, std::move(color_descriptor)); const size_t rot_array_size = n_vertices * 4 * sizeof(float); + float* rot_array = static_cast(malloc(rot_array_size)); if (geometry_.HasPointAttr("rot")) { - float* rot_array = static_cast(malloc(rot_array_size)); memcpy(rot_array, geometry_.GetPointAttr("rot").GetDataPtr(), rot_array_size); - VertexBuffer::BufferDescriptor rot_descriptor( - rot_array, rot_array_size, - GeometryBuffersBuilder::DeallocateBuffer); - vbuf->setBufferAt(engine, 3, std::move(rot_descriptor)); } else { - float* rot_array = static_cast(malloc(rot_array_size)); for (size_t i = 0; i < n_vertices * 4; ++i) { rot_array[i] = 1.f; } - VertexBuffer::BufferDescriptor rot_descriptor( + } + VertexBuffer::BufferDescriptor rot_descriptor( rot_array, rot_array_size, GeometryBuffersBuilder::DeallocateBuffer); - vbuf->setBufferAt(engine, 3, std::move(rot_descriptor)); - } + vbuf->setBufferAt(engine, 2, std::move(rot_descriptor)); - const size_t scale_array_size = n_vertices * 3 * sizeof(float); + const size_t scale_array_size = n_vertices * 4 * sizeof(float); float* scale_array = static_cast(malloc(scale_array_size)); + memset(scale_array, 0, scale_array_size); if (geometry_.HasPointAttr("scale")) { - const float* scale_src = static_cast( - geometry_.GetPointAttr("scale").GetDataPtr()); - memcpy(scale_array, scale_src, scale_array_size); - } else { - memset(scale_array, 0, scale_array_size); + float* scale_src = static_cast(geometry_.GetPointAttr("scale").GetDataPtr()); + for (size_t i = 0; i < n_vertices; i++) { + std::memcpy(scale_array + i * 4, scale_src + i * 3, 3*sizeof(float)); + } } VertexBuffer::BufferDescriptor scale_descriptor( scale_array, scale_array_size, GeometryBuffersBuilder::DeallocateBuffer); - vbuf->setBufferAt(engine, 4, std::move(scale_descriptor)); - - const size_t opacity_array_size = n_vertices * 1 * sizeof(float); - float* opacity_array = static_cast(malloc(opacity_array_size)); - if (geometry_.HasPointAttr("opacity")) { - const float* opacity_src = static_cast( - geometry_.GetPointAttr("opacity").GetDataPtr()); - memcpy(opacity_array, opacity_src, opacity_array_size); - } else { - memset(opacity_array, 0, opacity_array_size); - } - VertexBuffer::BufferDescriptor opacity_descriptor( - opacity_array, opacity_array_size, GeometryBuffersBuilder::DeallocateBuffer); - vbuf->setBufferAt(engine, 5, std::move(opacity_descriptor)); - - const size_t f_dc_array_size = n_vertices * 3 * sizeof(float); - float* f_dc_array = static_cast(malloc(f_dc_array_size)); - if (geometry_.HasPointAttr("f_dc")) { - const float* f_dc_src = static_cast( - geometry_.GetPointAttr("f_dc").GetDataPtr()); - memcpy(f_dc_array, f_dc_src, f_dc_array_size); - } else { - memset(f_dc_array, 0, f_dc_array_size); - } - VertexBuffer::BufferDescriptor f_dc_descriptor( - f_dc_array, f_dc_array_size, GeometryBuffersBuilder::DeallocateBuffer); - vbuf->setBufferAt(engine, 6, std::move(f_dc_descriptor)); - - const size_t uv_array_size = n_vertices * 2 * sizeof(float); - float* uv_array = static_cast(malloc(uv_array_size)); - if (geometry_.HasPointAttr("uv")) { - const float* uv_src = static_cast( - geometry_.GetPointAttr("uv").GetDataPtr()); - memcpy(uv_array, uv_src, uv_array_size); - } else if (geometry_.HasPointAttr("__visualization_scalar")) { - // Update in FilamentScene::UpdateGeometry(), too. - memset(uv_array, 0, uv_array_size); - auto vis_scalars = - geometry_.GetPointAttr("__visualization_scalar").Contiguous(); - const float* src = static_cast(vis_scalars.GetDataPtr()); - const size_t n = 2 * n_vertices; - for (size_t i = 0; i < n; i += 2) { - uv_array[i] = *src++; + vbuf->setBufferAt(engine, 3, std::move(scale_descriptor)); + + const size_t f_rest_array_size = n_vertices * 4 * sizeof(float); + const size_t custom_buffer_num = 6; + const size_t custom_buffer_start_index = 4; + for(size_t i = 0; i < custom_buffer_num; i++) { + float* f_rest_array = static_cast(malloc(f_rest_array_size)); + if (geometry_.HasPointAttr("f_rest")) { + float* f_rest_src = static_cast(geometry_.GetPointAttr("f_rest").GetDataPtr()); + memcpy(f_rest_array, f_rest_src + i * n_vertices * 4, f_rest_array_size); + } else { + memset(f_rest_array, 0, f_rest_array_size); } - } else { - memset(uv_array, 0, uv_array_size); + VertexBuffer::BufferDescriptor f_rest_descriptor( + f_rest_array, f_rest_array_size, GeometryBuffersBuilder::DeallocateBuffer); + vbuf->setBufferAt(engine, custom_buffer_start_index + i, std::move(f_rest_descriptor)); } - VertexBuffer::BufferDescriptor uv_descriptor( - uv_array, uv_array_size, GeometryBuffersBuilder::DeallocateBuffer); - vbuf->setBufferAt(engine, 7, std::move(uv_descriptor)); auto ib_handle = CreateIndexBuffer(n_vertices); From 9b76ad4e6d3257b7bd90fd64ed7091b43f8118dc Mon Sep 17 00:00:00 2001 From: Zhang Yang Date: Sat, 8 Mar 2025 11:39:48 +0800 Subject: [PATCH 04/28] update construct function --- .../filament/GaussianSplatBuffers.cpp | 46 +++++++++---------- 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp b/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp index a85a900a3b2..c37b70795db 100644 --- a/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp +++ b/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp @@ -89,6 +89,12 @@ struct ColoredVertex { TGaussianSplatBuffersBuilder::TGaussianSplatBuffersBuilder( const t::geometry::PointCloud& geometry) : geometry_(geometry) { + if (!geometry.IsGaussianSplat()) { + utility::LogWarning( + "TGaussianSplatBuffers is constructed for a geometry that is not GaussianSplat." + ); + } + // Make sure geometry is on CPU auto pts = geometry.GetPointPositions(); if (pts.IsCUDA()) { @@ -106,25 +112,17 @@ TGaussianSplatBuffersBuilder::TGaussianSplatBuffersBuilder( pts.GetDtype().ToString()); geometry_.GetPointPositions() = pts.To(core::Float32); } - if (geometry_.HasPointNormals() && - geometry_.GetPointNormals().GetDtype() != core::Float32) { - auto normals = geometry_.GetPointNormals(); - utility::LogWarning( - "Tensor point cloud normals must have DType of Float32 not {}. " - "Converting.", - normals.GetDtype().ToString()); - geometry_.GetPointNormals() = normals.To(core::Float32); - } - if (geometry_.HasPointColors() && - geometry_.GetPointColors().GetDtype() != core::Float32) { - auto colors = geometry_.GetPointColors(); - utility::LogWarning( - "Tensor point cloud colors must have DType of Float32 not {}. " - "Converting.", - colors.GetDtype().ToString()); - geometry_.GetPointColors() = colors.To(core::Float32); - if (colors.GetDtype() == core::UInt8) { - geometry_.GetPointColors() = geometry_.GetPointColors() / 255.0f; + + std::vector check_list = {"f_dc", "opacity", "rot", "scale", "f_rest"}; + for (const auto& check_item : check_list) { + if (geometry_.HasPointAttr(check_item) && + geometry_.GetPointAttr(check_item).GetDtype() != core::Float32) { + auto check_item_instance = geometry_.GetPointAttr(check_item); + utility::LogWarning( + "Tensor gaussian splat {} must have DType of Float32 not {}. " + "Converting.", + check_item, check_item_instance.GetDtype().ToString()); + geometry_.GetPointAttr(check_item) = check_item_instance.To(core::Float32); } } } @@ -141,7 +139,7 @@ GeometryBuffersBuilder::Buffers TGaussianSplatBuffersBuilder::ConstructBuffers() const auto& points = geometry_.GetPointPositions(); const size_t n_vertices = points.GetLength(); - // we usePOSITION for positions, COLOR for f_dc and opacity, TANGENTS for rot + // we use POSITION for positions, COLOR for f_dc and opacity, TANGENTS for rot // CUSTOM0 for scale, CUSTOM1-CUSTOM6 for f_rest. VertexBuffer* vbuf = VertexBuffer::Builder() .bufferCount(10) @@ -205,7 +203,7 @@ GeometryBuffersBuilder::Buffers TGaussianSplatBuffersBuilder::ConstructBuffers() const size_t rot_array_size = n_vertices * 4 * sizeof(float); float* rot_array = static_cast(malloc(rot_array_size)); if (geometry_.HasPointAttr("rot")) { - memcpy(rot_array, geometry_.GetPointAttr("rot").GetDataPtr(), + std::memcpy(rot_array, geometry_.GetPointAttr("rot").GetDataPtr(), rot_array_size); } else { for (size_t i = 0; i < n_vertices * 4; ++i) { @@ -219,7 +217,7 @@ GeometryBuffersBuilder::Buffers TGaussianSplatBuffersBuilder::ConstructBuffers() const size_t scale_array_size = n_vertices * 4 * sizeof(float); float* scale_array = static_cast(malloc(scale_array_size)); - memset(scale_array, 0, scale_array_size); + std::memset(scale_array, 0, scale_array_size); if (geometry_.HasPointAttr("scale")) { float* scale_src = static_cast(geometry_.GetPointAttr("scale").GetDataPtr()); for (size_t i = 0; i < n_vertices; i++) { @@ -237,9 +235,9 @@ GeometryBuffersBuilder::Buffers TGaussianSplatBuffersBuilder::ConstructBuffers() float* f_rest_array = static_cast(malloc(f_rest_array_size)); if (geometry_.HasPointAttr("f_rest")) { float* f_rest_src = static_cast(geometry_.GetPointAttr("f_rest").GetDataPtr()); - memcpy(f_rest_array, f_rest_src + i * n_vertices * 4, f_rest_array_size); + std::memcpy(f_rest_array, f_rest_src + i * n_vertices * 4, f_rest_array_size); } else { - memset(f_rest_array, 0, f_rest_array_size); + std::memset(f_rest_array, 0, f_rest_array_size); } VertexBuffer::BufferDescriptor f_rest_descriptor( f_rest_array, f_rest_array_size, GeometryBuffersBuilder::DeallocateBuffer); From b8ee9229a653c6d8740f2f2d4df6ca876730957e Mon Sep 17 00:00:00 2001 From: Zhang Yang Date: Sat, 8 Mar 2025 12:20:19 +0800 Subject: [PATCH 05/28] overwrite TGaussianSplatBuffersBuilder class --- .../filament/FilamentGeometryBuffersBuilder.h | 11 +- .../filament/GaussianSplatBuffers.cpp | 100 +----------------- 2 files changed, 3 insertions(+), 108 deletions(-) diff --git a/cpp/open3d/visualization/rendering/filament/FilamentGeometryBuffersBuilder.h b/cpp/open3d/visualization/rendering/filament/FilamentGeometryBuffersBuilder.h index 9fde9821420..1a0ab7ff56e 100644 --- a/cpp/open3d/visualization/rendering/filament/FilamentGeometryBuffersBuilder.h +++ b/cpp/open3d/visualization/rendering/filament/FilamentGeometryBuffersBuilder.h @@ -172,22 +172,15 @@ class TPointCloudBuffersBuilder : public GeometryBuffersBuilder { Buffers ConstructBuffers() override; filament::Box ComputeAABB() override; -private: +protected: t::geometry::PointCloud geometry_; }; -class TGaussianSplatBuffersBuilder : public GeometryBuffersBuilder { +class TGaussianSplatBuffersBuilder : public TPointCloudBuffersBuilder { public: explicit TGaussianSplatBuffersBuilder(const t::geometry::PointCloud& geometry); - filament::RenderableManager::PrimitiveType GetPrimitiveType() - const override; - Buffers ConstructBuffers() override; - filament::Box ComputeAABB() override; - -private: - t::geometry::PointCloud geometry_; }; class TLineSetBuffersBuilder : public GeometryBuffersBuilder { diff --git a/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp b/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp index c37b70795db..8fa679654c7 100644 --- a/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp +++ b/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp @@ -37,81 +37,14 @@ using namespace filament; namespace open3d { namespace visualization { namespace rendering { - -namespace { -struct ColoredVertex { - math::float3 position = {0.f, 0.f, 0.f}; - // Default to mid-gray which provides good separation from the two most - // common background colors: white and black. Otherwise, point clouds - // without per-vertex colors may appear is if they are not rendering because - // they blend with the background. - math::float4 color = {0.5f, 0.5f, 0.5f, 1.f}; - math::quatf tangent = {0.f, 0.f, 0.f, 1.f}; - math::float2 uv = {0.f, 0.f}; - - static std::uint32_t GetPositionOffset() { - return offsetof(ColoredVertex, position); - } - static std::uint32_t GetColorOffset() { - return offsetof(ColoredVertex, color); - } - static std::uint32_t GetTangentOffset() { - return offsetof(ColoredVertex, tangent); - } - static std::uint32_t GetUVOffset() { return offsetof(ColoredVertex, uv); } - void SetVertexPosition(const Eigen::Vector3d& pos) { - auto float_pos = pos.cast(); - position.x = float_pos(0); - position.y = float_pos(1); - position.z = float_pos(2); - } - - float sRGBToLinear(float color) { - return color <= 0.04045f ? color / 12.92f - : pow((color + 0.055f) / 1.055f, 2.4f); - } - - void SetVertexColor(const Eigen::Vector3d& c, bool adjust_for_srgb) { - auto float_color = c.cast(); - if (adjust_for_srgb) { - color.x = sRGBToLinear(float_color(0)); - color.y = sRGBToLinear(float_color(1)); - color.z = sRGBToLinear(float_color(2)); - } else { - color.x = float_color(0); - color.y = float_color(1); - color.z = float_color(2); - } - } -}; -} // namespace - TGaussianSplatBuffersBuilder::TGaussianSplatBuffersBuilder( const t::geometry::PointCloud& geometry) - : geometry_(geometry) { + : TPointCloudBuffersBuilder(geometry) { if (!geometry.IsGaussianSplat()) { utility::LogWarning( "TGaussianSplatBuffers is constructed for a geometry that is not GaussianSplat." ); } - - // Make sure geometry is on CPU - auto pts = geometry.GetPointPositions(); - if (pts.IsCUDA()) { - utility::LogWarning( - "GPU resident point clouds are not currently supported for " - "visualization. Copying data to CPU."); - geometry_ = geometry.To(core::Device("CPU:0")); - } - - // Now make sure data types are Float32 - if (pts.GetDtype() != core::Float32) { - utility::LogWarning( - "Tensor point cloud points must have DType of Float32 not {}. " - "Converting.", - pts.GetDtype().ToString()); - geometry_.GetPointPositions() = pts.To(core::Float32); - } std::vector check_list = {"f_dc", "opacity", "rot", "scale", "f_rest"}; for (const auto& check_item : check_list) { @@ -127,11 +60,6 @@ TGaussianSplatBuffersBuilder::TGaussianSplatBuffersBuilder( } } -RenderableManager::PrimitiveType TGaussianSplatBuffersBuilder::GetPrimitiveType() - const { - return RenderableManager::PrimitiveType::POINTS; -} - GeometryBuffersBuilder::Buffers TGaussianSplatBuffersBuilder::ConstructBuffers() { auto& engine = EngineInstance::GetInstance(); auto& resource_mgr = EngineInstance::GetResourceManager(); @@ -254,32 +182,6 @@ GeometryBuffersBuilder::Buffers TGaussianSplatBuffersBuilder::ConstructBuffers() return std::make_tuple(vb_handle, ib_handle, downsampled_handle); } - -filament::Box TGaussianSplatBuffersBuilder::ComputeAABB() { - auto min_bounds = geometry_.GetMinBound(); - auto max_bounds = geometry_.GetMaxBound(); - auto* min_bounds_float = min_bounds.GetDataPtr(); - auto* max_bounds_float = max_bounds.GetDataPtr(); - - filament::math::float3 min(min_bounds_float[0], min_bounds_float[1], - min_bounds_float[2]); - filament::math::float3 max(max_bounds_float[0], max_bounds_float[1], - max_bounds_float[2]); - - Box aabb; - aabb.set(min, max); - if (aabb.isEmpty()) { - min.x -= 0.1f; - min.y -= 0.1f; - min.z -= 0.1f; - max.x += 0.1f; - max.y += 0.1f; - max.z += 0.1f; - aabb.set(min, max); - } - return aabb; -} - } // namespace rendering } // namespace visualization } // namespace open3d From 0e8f9b0b6d302ef3b8b864894240ddda18b69871 Mon Sep 17 00:00:00 2001 From: Zhang Yang Date: Sat, 8 Mar 2025 12:30:11 +0800 Subject: [PATCH 06/28] add check logic for gaussian --- cpp/open3d/visualization/visualizer/O3DVisualizer.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cpp/open3d/visualization/visualizer/O3DVisualizer.cpp b/cpp/open3d/visualization/visualizer/O3DVisualizer.cpp index 37fb415e1f4..dae6bf9fbae 100644 --- a/cpp/open3d/visualization/visualizer/O3DVisualizer.cpp +++ b/cpp/open3d/visualization/visualizer/O3DVisualizer.cpp @@ -861,6 +861,7 @@ Ctrl-alt-click to polygon select)"; } else { // branch only applies to geometries bool has_colors = false; bool has_normals = false; + bool is_gaussian_splat = false; auto cloud = std::dynamic_pointer_cast(geom); auto lines = std::dynamic_pointer_cast(geom); @@ -887,6 +888,7 @@ Ctrl-alt-click to polygon select)"; has_colors = !cloud->colors_.empty(); has_normals = !cloud->normals_.empty(); } else if (t_cloud) { + if (t_cloud->IsGaussianSplat()) is_gaussian_splat = true; has_colors = t_cloud->HasPointColors(); has_normals = t_cloud->HasPointNormals(); } else if (lines) { @@ -931,9 +933,11 @@ Ctrl-alt-click to polygon select)"; if (has_normals) { mat.base_color = {1.0f, 1.0f, 1.0f, 1.0f}; mat.shader = kShaderLit; - mat.shader = kShaderGaussianSplat; is_default_color = false; } + if (is_gaussian_splat) { + mat.shader = kShaderGaussianSplat; + } mat.point_size = ConvertToScaledPixels(ui_state_.point_size); // If T Geometry has a valid material convert it to MaterialRecord From 2b25cb06df3749a4bf94e471a7e4250fa1c9a86f Mon Sep 17 00:00:00 2001 From: Yujie Wang Date: Sat, 8 Mar 2025 13:02:09 +0800 Subject: [PATCH 07/28] remove .mat related files will create a new PR for this --- cpp/open3d/visualization/gui/CMakeLists.txt | 1 - .../visualization/gui/Materials/gaussian.mat | 65 ------------------- .../filament/FilamentResourceManager.cpp | 2 +- 3 files changed, 1 insertion(+), 67 deletions(-) delete mode 100644 cpp/open3d/visualization/gui/Materials/gaussian.mat diff --git a/cpp/open3d/visualization/gui/CMakeLists.txt b/cpp/open3d/visualization/gui/CMakeLists.txt index 76280a692e3..a3901dc2b0a 100644 --- a/cpp/open3d/visualization/gui/CMakeLists.txt +++ b/cpp/open3d/visualization/gui/CMakeLists.txt @@ -98,7 +98,6 @@ open3d_add_compiled_materials(materials Materials/unlitLine.mat Materials/unlitPolygonOffset.mat Materials/unlitSolidColor.mat - Materials/gaussian.mat ) # Source group for Visual Studio diff --git a/cpp/open3d/visualization/gui/Materials/gaussian.mat b/cpp/open3d/visualization/gui/Materials/gaussian.mat deleted file mode 100644 index 3577d35a77e..00000000000 --- a/cpp/open3d/visualization/gui/Materials/gaussian.mat +++ /dev/null @@ -1,65 +0,0 @@ -material { - name : defaultLit, - shadingModel : lit, - doubleSided : true, - - parameters : [ - { type : float3, name : baseColor }, - { type : float, name : baseMetallic }, - { type : float, name : baseRoughness }, - { type : float, name : reflectance }, - { type : float, name : clearCoat }, - { type : float, name : clearCoatRoughness }, - { type : float, name : anisotropy }, - { type : float, name : pointSize }, - { type : sampler2d, name : albedo }, - { type : sampler2d, name : ao_rough_metalMap }, - { type : sampler2d, name : normalMap }, - { type : sampler2d, name : reflectanceMap }, -// NOTE: Disable clear coat textures to avoid Filament warning about exceeding -// 16 sampler limit. There is currently no supported standard format that -// supports these textures so they don't get used any way. Once the shader has -// been updated to combine roughness/metallic/reflectance/ao these textures -// will be reenabled. -// -// { type : sampler2d, name : clearCoatMap }, -// { type : sampler2d, name : clearCoatRoughnessMap }, - { type : sampler2d, name : anisotropyMap } - ], - requires : [ - color, uv0 - ] -} - -vertex { - void materialVertex(inout MaterialVertexInputs material) { - gl_PointSize = materialParams.pointSize; - } -} - -fragment { - void material(inout MaterialInputs material) { - // Modifications to normal must be done before prepareMaterial call - material.normal = texture(materialParams_normalMap, getUV0()).xyz * 2.0 - 1.0; - material.normal.y *= -1.0; // NOTE: including this line because Filament samples do - // Flip normals for point clouds in case of inspection mode - // if (dot(getWorldGeometricNormalVector(), getWorldViewVector()) < 0.0) - // material.normal = -material.normal; - prepareMaterial(material); - - material.baseColor.rgb = getColor().rgb * materialParams.baseColor * texture(materialParams_albedo, getUV0()).rgb; - float3 ao_rough_metal = texture(materialParams_ao_rough_metalMap, getUV0()).rgb; - material.metallic = materialParams.baseMetallic * ao_rough_metal.b; - material.roughness = materialParams.baseRoughness * ao_rough_metal.g; - material.ambientOcclusion = ao_rough_metal.r; - //material.reflectance = materialParams.reflectance; - //material.anisotropy = materialParams.anisotropy; - material.reflectance = materialParams.reflectance * texture(materialParams_reflectanceMap, getUV0()).r; - // See note above about following disabled textures - material.clearCoat = materialParams.clearCoat; - //material.clearCoat = materialParams.clearCoat * texture(materialParams_clearCoatMap, getUV0()).r; - material.clearCoatRoughness = materialParams.clearCoatRoughness; - //material.clearCoatRoughness = materialParams.clearCoatRoughness * texture(materialParams_clearCoatRoughnessMap, getUV0()).r; - material.anisotropy = materialParams.anisotropy * texture(materialParams_anisotropyMap, getUV0()).r; - } -} diff --git a/cpp/open3d/visualization/rendering/filament/FilamentResourceManager.cpp b/cpp/open3d/visualization/rendering/filament/FilamentResourceManager.cpp index a9248975caf..a0504b05f58 100644 --- a/cpp/open3d/visualization/rendering/filament/FilamentResourceManager.cpp +++ b/cpp/open3d/visualization/rendering/filament/FilamentResourceManager.cpp @@ -1019,7 +1019,7 @@ void FilamentResourceManager::LoadDefaults() { lit_mat->setDefaultParameter("anisotropyMap", texture, default_sampler); materials_[kDefaultLit] = BoxResource(lit_mat, engine_); - const auto gaussian_path = resource_root + "/gaussian.filamat"; + const auto gaussian_path = resource_root + "/gaussianSplat.filamat"; auto gaussian_mat = LoadMaterialFromFile(gaussian_path, engine_); gaussian_mat->setDefaultParameter("baseColor", filament::RgbType::sRGB, default_color); From 84617010ddae900b6c4ffb230d4731a004db984a Mon Sep 17 00:00:00 2001 From: Yujie Wang Date: Sat, 8 Mar 2025 13:30:13 +0800 Subject: [PATCH 08/28] add C++ API doc --- cpp/open3d/t/geometry/PointCloud.h | 6 +++++ .../filament/FilamentGeometryBuffersBuilder.h | 23 +++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/cpp/open3d/t/geometry/PointCloud.h b/cpp/open3d/t/geometry/PointCloud.h index e3dd1fbf391..1293de44df1 100644 --- a/cpp/open3d/t/geometry/PointCloud.h +++ b/cpp/open3d/t/geometry/PointCloud.h @@ -738,6 +738,12 @@ class PointCloud : public Geometry, public DrawableGeometry { std::vector metrics = {Metric::ChamferDistance}, MetricParameters params = MetricParameters()) const; + /// \brief Checks if the point cloud is formatted for Gaussian Splatting. + /// + /// This function verifies that the point attribute container contains all the required keys + /// ("f_dc", "opacity", "rot", and "scale") necessary for Gaussian Splatting. If any of these keys + /// are missing, the function returns \c false; otherwise, it returns \c true. + /// \return \c true if all required attributes are present, \c false otherwise. bool IsGaussianSplat() const; protected: core::Device device_ = core::Device("CPU:0"); diff --git a/cpp/open3d/visualization/rendering/filament/FilamentGeometryBuffersBuilder.h b/cpp/open3d/visualization/rendering/filament/FilamentGeometryBuffersBuilder.h index 1a0ab7ff56e..e8130697853 100644 --- a/cpp/open3d/visualization/rendering/filament/FilamentGeometryBuffersBuilder.h +++ b/cpp/open3d/visualization/rendering/filament/FilamentGeometryBuffersBuilder.h @@ -178,8 +178,31 @@ class TPointCloudBuffersBuilder : public GeometryBuffersBuilder { class TGaussianSplatBuffersBuilder : public TPointCloudBuffersBuilder { public: + /// \brief Constructs a TGaussianSplatBuffersBuilder object. + /// + /// Initializes the Gaussian Splat buffers from the provided \p geometry and ensures that + /// all necessary attributes are present and correctly formatted. If the geometry is not a + /// Gaussian Splat, a warning is issued. Additionally, attributes like "f_dc", "opacity", + /// "rot", "scale", and "f_rest" are checked for their data type, and converted to Float32 + /// if they are not already in that format. explicit TGaussianSplatBuffersBuilder(const t::geometry::PointCloud& geometry); + /// \brief Constructs vertex and index buffers for Gaussian Splat rendering. + /// + /// This function creates and configures GPU buffers to represent a Gaussian Splat + /// point cloud. It extracts attributes like positions, colors, rotation, scale, + /// and spherical harmonics coefficients from the provided \ref geometry_ and + /// organizes them into separate vertex buffer attributes. + /// + /// The vertex buffer contains the following attributes: + /// - POSITION: Vertex positions (FLOAT3) + /// - COLOR: DC component and opacity (FLOAT4) + /// - TANGENTS: Rotation quaternion (FLOAT4) + /// - CUSTOM0: Scale (FLOAT4) + /// - CUSTOM1 to CUSTOM6: SH coefficients (FLOAT4) + /// + /// Each attribute is checked and converted to the expected data type if necessary, + /// and missing attributes are initialized with default values. Buffers ConstructBuffers() override; }; From 46b7657c3a846055a69d323021b64462f22a3a85 Mon Sep 17 00:00:00 2001 From: Zhang Yang Date: Wed, 12 Mar 2025 14:20:33 +0800 Subject: [PATCH 09/28] update gaussian buffer Co-authored-by: hanyin-intel han.yin@intel.com --- .../visualization/rendering/MaterialRecord.h | 3 + .../rendering/filament/FilamentScene.cpp | 29 ++++++++- .../rendering/filament/FilamentScene.h | 1 + .../filament/GaussianSplatBuffers.cpp | 64 +++++++++---------- 4 files changed, 63 insertions(+), 34 deletions(-) diff --git a/cpp/open3d/visualization/rendering/MaterialRecord.h b/cpp/open3d/visualization/rendering/MaterialRecord.h index 6822cbaf1ff..61970ae5b90 100644 --- a/cpp/open3d/visualization/rendering/MaterialRecord.h +++ b/cpp/open3d/visualization/rendering/MaterialRecord.h @@ -83,6 +83,9 @@ struct MaterialRecord { // Infinite ground plane float ground_plane_axis = 0.f; // 0: XZ; >0: XY; <0: YZ + // This is only used in gaussian splat. + int sh_degree = 2; + // Generic material properties std::unordered_map generic_params; std::unordered_map generic_imgs; diff --git a/cpp/open3d/visualization/rendering/filament/FilamentScene.cpp b/cpp/open3d/visualization/rendering/filament/FilamentScene.cpp index 9e364f91f1f..0afb898538a 100644 --- a/cpp/open3d/visualization/rendering/filament/FilamentScene.cpp +++ b/cpp/open3d/visualization/rendering/filament/FilamentScene.cpp @@ -847,6 +847,32 @@ void FilamentScene::UpdateDefaultLit(GeometryMaterialInstance& geom_mi) { .Finish(); } +void FilamentScene::UpdateGaussianSplat(GeometryMaterialInstance& geom_mi) { + auto& material = geom_mi.properties; + auto& maps = geom_mi.maps; + + renderer_.ModifyMaterial(geom_mi.mat_instance) + .SetColor("baseColor", material.base_color, false) + .SetParameter("pointSize", material.point_size) + .SetParameter("baseRoughness", material.base_roughness) + .SetParameter("baseMetallic", material.base_metallic) + .SetParameter("reflectance", material.base_reflectance) + .SetParameter("clearCoat", material.base_clearcoat) + .SetParameter("clearCoatRoughness", + material.base_clearcoat_roughness) + .SetParameter("anisotropy", material.base_anisotropy) + .SetParameter("shDegree", material.sh_degree) + .SetTexture("albedo", maps.albedo_map, + rendering::TextureSamplerParameters::Pretty()) + .SetTexture("normalMap", maps.normal_map, + rendering::TextureSamplerParameters::Pretty()) + .SetTexture("ao_rough_metalMap", maps.ao_rough_metal_map, + rendering::TextureSamplerParameters::Pretty()) + .SetTexture("reflectanceMap", maps.reflectance_map, + rendering::TextureSamplerParameters::Pretty()) + .Finish(); +} + void FilamentScene::UpdateDefaultLitSSR(GeometryMaterialInstance& geom_mi) { auto& material = geom_mi.properties; auto& maps = geom_mi.maps; @@ -1133,8 +1159,7 @@ void FilamentScene::UpdateMaterialProperties(RenderableGeometry& geom) { } else if (props.shader == "unlitPolygonOffset") { UpdateUnlitPolygonOffsetShader(geom.mat); } else if (props.shader == "gaussianSplat") { - UpdateDefaultLit(geom.mat); - // need to make changes here, just update the parameter + UpdateGaussianSplat(geom.mat); } else { utility::LogWarning("'{}' is not a valid shader", props.shader); } diff --git a/cpp/open3d/visualization/rendering/filament/FilamentScene.h b/cpp/open3d/visualization/rendering/filament/FilamentScene.h index 4556f895441..f6682851d30 100644 --- a/cpp/open3d/visualization/rendering/filament/FilamentScene.h +++ b/cpp/open3d/visualization/rendering/filament/FilamentScene.h @@ -305,6 +305,7 @@ class FilamentScene : public Scene { bool shader_only = false); void UpdateMaterialProperties(RenderableGeometry& geom); void UpdateDefaultLit(GeometryMaterialInstance& geom_mi); + void UpdateGaussianSplat(GeometryMaterialInstance& geom_mi); void UpdateDefaultLitSSR(GeometryMaterialInstance& geom_mi); void UpdateDefaultUnlit(GeometryMaterialInstance& geom_mi); void UpdateNormalShader(GeometryMaterialInstance& geom_mi); diff --git a/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp b/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp index 8fa679654c7..b802a19024b 100644 --- a/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp +++ b/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp @@ -67,8 +67,8 @@ GeometryBuffersBuilder::Buffers TGaussianSplatBuffersBuilder::ConstructBuffers() const auto& points = geometry_.GetPointPositions(); const size_t n_vertices = points.GetLength(); - // we use POSITION for positions, COLOR for f_dc and opacity, TANGENTS for rot - // CUSTOM0 for scale, CUSTOM1-CUSTOM6 for f_rest. + // we use POSITION for positions, COLOR for scale, TANGENTS for rot + // CUSTOM0 for f_dc and opacity, CUSTOM1-CUSTOM6 for f_rest. VertexBuffer* vbuf = VertexBuffer::Builder() .bufferCount(10) .vertexCount(uint32_t(n_vertices)) @@ -81,17 +81,17 @@ GeometryBuffersBuilder::Buffers TGaussianSplatBuffersBuilder::ConstructBuffers() .attribute(VertexAttribute::CUSTOM0, 3, VertexBuffer::AttributeType::FLOAT4) .attribute(VertexAttribute::CUSTOM1, 4, - VertexBuffer::AttributeType::FLOAT4) + VertexBuffer::AttributeType::FLOAT4) .attribute(VertexAttribute::CUSTOM2, 5, - VertexBuffer::AttributeType::FLOAT4) + VertexBuffer::AttributeType::FLOAT4) .attribute(VertexAttribute::CUSTOM3, 6, - VertexBuffer::AttributeType::FLOAT4) + VertexBuffer::AttributeType::FLOAT4) .attribute(VertexAttribute::CUSTOM4, 7, - VertexBuffer::AttributeType::FLOAT4) + VertexBuffer::AttributeType::FLOAT4) .attribute(VertexAttribute::CUSTOM5, 8, - VertexBuffer::AttributeType::FLOAT4) + VertexBuffer::AttributeType::FLOAT4) .attribute(VertexAttribute::CUSTOM6, 9, - VertexBuffer::AttributeType::FLOAT4) + VertexBuffer::AttributeType::FLOAT4) .build(engine); VertexBufferHandle vb_handle; @@ -109,24 +109,18 @@ GeometryBuffersBuilder::Buffers TGaussianSplatBuffersBuilder::ConstructBuffers() GeometryBuffersBuilder::DeallocateBuffer); vbuf->setBufferAt(engine, 0, std::move(pts_descriptor)); - const size_t color_array_size = n_vertices * 4 * sizeof(float); - float* color_array = static_cast(malloc(color_array_size)); - if (geometry_.HasPointAttr("f_dc") && geometry_.HasPointAttr("opacity")) { - float* f_dc_ptr = static_cast(geometry_.GetPointAttr("f_dc").GetDataPtr()); - float* opacity_ptr = static_cast(geometry_.GetPointAttr("opacity").GetDataPtr()); + const size_t scale_array_size = n_vertices * 4 * sizeof(float); + float* scale_array = static_cast(malloc(scale_array_size)); + std::memset(scale_array, 0, scale_array_size); + if (geometry_.HasPointAttr("scale")) { + float* scale_src = static_cast(geometry_.GetPointAttr("scale").GetDataPtr()); for (size_t i = 0; i < n_vertices; i++) { - std::memcpy(color_array + i * 4, f_dc_ptr + i * 3, 3 * sizeof(float)); - std::memcpy(color_array + i * 4 + 3, opacity_ptr + i, sizeof(float)); - } - } else { - for (size_t i = 0; i < n_vertices * 4; i++) { - color_array[i] = 1.f; + std::memcpy(scale_array + i * 4, scale_src + i * 3, 3*sizeof(float)); } } - VertexBuffer::BufferDescriptor color_descriptor( - color_array, color_array_size, - GeometryBuffersBuilder::DeallocateBuffer); - vbuf->setBufferAt(engine, 1, std::move(color_descriptor)); + VertexBuffer::BufferDescriptor scale_descriptor( + scale_array, scale_array_size, GeometryBuffersBuilder::DeallocateBuffer); + vbuf->setBufferAt(engine, 1, std::move(scale_descriptor)); const size_t rot_array_size = n_vertices * 4 * sizeof(float); float* rot_array = static_cast(malloc(rot_array_size)); @@ -143,18 +137,24 @@ GeometryBuffersBuilder::Buffers TGaussianSplatBuffersBuilder::ConstructBuffers() GeometryBuffersBuilder::DeallocateBuffer); vbuf->setBufferAt(engine, 2, std::move(rot_descriptor)); - const size_t scale_array_size = n_vertices * 4 * sizeof(float); - float* scale_array = static_cast(malloc(scale_array_size)); - std::memset(scale_array, 0, scale_array_size); - if (geometry_.HasPointAttr("scale")) { - float* scale_src = static_cast(geometry_.GetPointAttr("scale").GetDataPtr()); + const size_t color_array_size = n_vertices * 4 * sizeof(float); + float* color_array = static_cast(malloc(color_array_size)); + if (geometry_.HasPointAttr("f_dc") && geometry_.HasPointAttr("opacity")) { + float* f_dc_ptr = static_cast(geometry_.GetPointAttr("f_dc").GetDataPtr()); + float* opacity_ptr = static_cast(geometry_.GetPointAttr("opacity").GetDataPtr()); for (size_t i = 0; i < n_vertices; i++) { - std::memcpy(scale_array + i * 4, scale_src + i * 3, 3*sizeof(float)); + std::memcpy(color_array + i * 4, f_dc_ptr + i * 3, 3 * sizeof(float)); + std::memcpy(color_array + i * 4 + 3, opacity_ptr + i, sizeof(float)); + } + } else { + for (size_t i = 0; i < n_vertices * 4; i++) { + color_array[i] = 1.f; } } - VertexBuffer::BufferDescriptor scale_descriptor( - scale_array, scale_array_size, GeometryBuffersBuilder::DeallocateBuffer); - vbuf->setBufferAt(engine, 3, std::move(scale_descriptor)); + VertexBuffer::BufferDescriptor color_descriptor( + color_array, color_array_size, + GeometryBuffersBuilder::DeallocateBuffer); + vbuf->setBufferAt(engine, 3, std::move(color_descriptor)); const size_t f_rest_array_size = n_vertices * 4 * sizeof(float); const size_t custom_buffer_num = 6; From 34d49a8a17256218e526c5e3e398992b78f3b347 Mon Sep 17 00:00:00 2001 From: Zhang Yang Date: Wed, 12 Mar 2025 19:22:46 +0800 Subject: [PATCH 10/28] remove unnecessary, if-else --- .../filament/GaussianSplatBuffers.cpp | 40 +++++++------------ 1 file changed, 14 insertions(+), 26 deletions(-) diff --git a/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp b/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp index b802a19024b..8e85dd29336 100644 --- a/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp +++ b/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp @@ -112,11 +112,9 @@ GeometryBuffersBuilder::Buffers TGaussianSplatBuffersBuilder::ConstructBuffers() const size_t scale_array_size = n_vertices * 4 * sizeof(float); float* scale_array = static_cast(malloc(scale_array_size)); std::memset(scale_array, 0, scale_array_size); - if (geometry_.HasPointAttr("scale")) { - float* scale_src = static_cast(geometry_.GetPointAttr("scale").GetDataPtr()); - for (size_t i = 0; i < n_vertices; i++) { - std::memcpy(scale_array + i * 4, scale_src + i * 3, 3*sizeof(float)); - } + float* scale_src = geometry_.GetPointAttr("scale").GetDataPtr(); + for (size_t i = 0; i < n_vertices; i++) { + std::memcpy(scale_array + i * 4, scale_src + i * 3, 3*sizeof(float)); } VertexBuffer::BufferDescriptor scale_descriptor( scale_array, scale_array_size, GeometryBuffersBuilder::DeallocateBuffer); @@ -124,32 +122,22 @@ GeometryBuffersBuilder::Buffers TGaussianSplatBuffersBuilder::ConstructBuffers() const size_t rot_array_size = n_vertices * 4 * sizeof(float); float* rot_array = static_cast(malloc(rot_array_size)); - if (geometry_.HasPointAttr("rot")) { - std::memcpy(rot_array, geometry_.GetPointAttr("rot").GetDataPtr(), - rot_array_size); - } else { - for (size_t i = 0; i < n_vertices * 4; ++i) { - rot_array[i] = 1.f; - } - } + std::memset(rot_array, 0, rot_array_size); + std::memcpy(rot_array, geometry_.GetPointAttr("rot").GetDataPtr(), + rot_array_size); VertexBuffer::BufferDescriptor rot_descriptor( rot_array, rot_array_size, GeometryBuffersBuilder::DeallocateBuffer); vbuf->setBufferAt(engine, 2, std::move(rot_descriptor)); const size_t color_array_size = n_vertices * 4 * sizeof(float); - float* color_array = static_cast(malloc(color_array_size)); - if (geometry_.HasPointAttr("f_dc") && geometry_.HasPointAttr("opacity")) { - float* f_dc_ptr = static_cast(geometry_.GetPointAttr("f_dc").GetDataPtr()); - float* opacity_ptr = static_cast(geometry_.GetPointAttr("opacity").GetDataPtr()); - for (size_t i = 0; i < n_vertices; i++) { - std::memcpy(color_array + i * 4, f_dc_ptr + i * 3, 3 * sizeof(float)); - std::memcpy(color_array + i * 4 + 3, opacity_ptr + i, sizeof(float)); - } - } else { - for (size_t i = 0; i < n_vertices * 4; i++) { - color_array[i] = 1.f; - } + float* color_array = static_cast(malloc(color_array_size)); + std::memset(color_array, 0, color_array_size); + float* f_dc_ptr = geometry_.GetPointAttr("f_dc").GetDataPtr(); + float* opacity_ptr = geometry_.GetPointAttr("opacity").GetDataPtr(); + for (size_t i = 0; i < n_vertices; i++) { + std::memcpy(color_array + i * 4, f_dc_ptr + i * 3, 3 * sizeof(float)); + std::memcpy(color_array + i * 4 + 3, opacity_ptr + i, sizeof(float)); } VertexBuffer::BufferDescriptor color_descriptor( color_array, color_array_size, @@ -162,7 +150,7 @@ GeometryBuffersBuilder::Buffers TGaussianSplatBuffersBuilder::ConstructBuffers() for(size_t i = 0; i < custom_buffer_num; i++) { float* f_rest_array = static_cast(malloc(f_rest_array_size)); if (geometry_.HasPointAttr("f_rest")) { - float* f_rest_src = static_cast(geometry_.GetPointAttr("f_rest").GetDataPtr()); + float* f_rest_src = geometry_.GetPointAttr("f_rest").GetDataPtr(); std::memcpy(f_rest_array, f_rest_src + i * n_vertices * 4, f_rest_array_size); } else { std::memset(f_rest_array, 0, f_rest_array_size); From dc68c3c6e71efe45620a7d461e236e1ea995c306 Mon Sep 17 00:00:00 2001 From: Zhang Yang Date: Thu, 13 Mar 2025 10:08:04 +0800 Subject: [PATCH 11/28] use template --- .../visualization/rendering/filament/PointCloudBuffers.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cpp/open3d/visualization/rendering/filament/PointCloudBuffers.cpp b/cpp/open3d/visualization/rendering/filament/PointCloudBuffers.cpp index 2d0c1295ed6..d14ac5394bb 100644 --- a/cpp/open3d/visualization/rendering/filament/PointCloudBuffers.cpp +++ b/cpp/open3d/visualization/rendering/filament/PointCloudBuffers.cpp @@ -434,15 +434,14 @@ GeometryBuffersBuilder::Buffers TPointCloudBuffersBuilder::ConstructBuffers() { const size_t uv_array_size = n_vertices * 2 * sizeof(float); float* uv_array = static_cast(malloc(uv_array_size)); if (geometry_.HasPointAttr("uv")) { - const float* uv_src = static_cast( - geometry_.GetPointAttr("uv").GetDataPtr()); + const float* uv_src = geometry_.GetPointAttr("uv").GetDataPtr(); memcpy(uv_array, uv_src, uv_array_size); } else if (geometry_.HasPointAttr("__visualization_scalar")) { // Update in FilamentScene::UpdateGeometry(), too. memset(uv_array, 0, uv_array_size); auto vis_scalars = geometry_.GetPointAttr("__visualization_scalar").Contiguous(); - const float* src = static_cast(vis_scalars.GetDataPtr()); + const float* src = vis_scalars.GetDataPtr(); const size_t n = 2 * n_vertices; for (size_t i = 0; i < n; i += 2) { uv_array[i] = *src++; From e80c591c14e3ac89e1b4fe14f424cf1b7558ae5c Mon Sep 17 00:00:00 2001 From: Zhang Yang Date: Thu, 13 Mar 2025 14:09:16 +0800 Subject: [PATCH 12/28] add PointCloud::GaussianSplatGetSHOrder() function --- cpp/open3d/t/geometry/PointCloud.cpp | 62 ++++++++++++++++++++++++++++ cpp/open3d/t/geometry/PointCloud.h | 5 +++ 2 files changed, 67 insertions(+) diff --git a/cpp/open3d/t/geometry/PointCloud.cpp b/cpp/open3d/t/geometry/PointCloud.cpp index d296570fe77..7ae5ab4de7f 100644 --- a/cpp/open3d/t/geometry/PointCloud.cpp +++ b/cpp/open3d/t/geometry/PointCloud.cpp @@ -1379,6 +1379,68 @@ bool PointCloud::IsGaussianSplat() const { return true; } +int PointCloud::GaussianSplatGetSHOrder() const { + if (point_attr_.find("f_rest") == point_attr_.end()) { + return 0; + } else { + const core::Tensor & f_rest = GetPointAttr("f_rest"); + auto shape = f_rest.GetShape(); + size_t num_dims = shape.size(); + + int dim_of_degree = shape[1]; + if (num_dims == 2) { + if (dim_of_degree % 3 != 0) { + utility::LogError("SH coeffs are in shape [N, Nc], but the dimension of Nc cannot be divided by 3"); + } + dim_of_degree = dim_of_degree / 3; + } + + std::optional degree = findPositiveIntegerSolution(dim_of_degree); + if (degree.has_value()) { + if (degree.value() > 2) { + utility::LogWarning("The SH coeffs degree higher than 2 is not supported for now."); + } + return degree.value(); + } else { + utility::LogError("The number of SH coeffs is not a valid value."); + } + return -1; + } +} + +std::optional PointCloud::findPositiveIntegerSolution(int x) const { + // Coefficients of the quadratic equation n^2 + 2n - x = 0 + int a = 1; + int b = 2; + int c = -x; + + // Calculate the discriminant + int discriminant = b * b - 4 * a * c; + + // Check if the discriminant is non-negative and a perfect square + if (discriminant < 0) { + return std::nullopt; + } + + int sqrt_discriminant = static_cast(std::sqrt(discriminant)); + if (sqrt_discriminant * sqrt_discriminant != discriminant) { + return std::nullopt; + } + + // Calculate the two possible solutions for n + int n1 = (-b + sqrt_discriminant) / (2 * a); + int n2 = (-b - sqrt_discriminant) / (2 * a); + + // Check if either solution is a valid positive integer + // There is definitely a positive result because x is positive. + if (n1 > 0 && n1 * (n1 + 2) == x) { + return n1; + } else if (n2 > 0 && n2 * (n2 + 2) == x) { + return n2; + } + + return std::nullopt; +} } // namespace geometry } // namespace t } // namespace open3d diff --git a/cpp/open3d/t/geometry/PointCloud.h b/cpp/open3d/t/geometry/PointCloud.h index 1293de44df1..35861482294 100644 --- a/cpp/open3d/t/geometry/PointCloud.h +++ b/cpp/open3d/t/geometry/PointCloud.h @@ -745,9 +745,14 @@ class PointCloud : public Geometry, public DrawableGeometry { /// are missing, the function returns \c false; otherwise, it returns \c true. /// \return \c true if all required attributes are present, \c false otherwise. bool IsGaussianSplat() const; + + int GaussianSplatGetSHOrder() const; protected: core::Device device_ = core::Device("CPU:0"); TensorMap point_attr_; + +private: + std::optional findPositiveIntegerSolution(int x) const; }; } // namespace geometry From 530f39b4072848e924acb87278e883a883b365f8 Mon Sep 17 00:00:00 2001 From: Zhang Yang Date: Thu, 13 Mar 2025 16:06:55 +0800 Subject: [PATCH 13/28] allocate buffer dynamically --- .../filament/GaussianSplatBuffers.cpp | 64 ++++++++++++------- 1 file changed, 40 insertions(+), 24 deletions(-) diff --git a/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp b/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp index 8e85dd29336..45fc00a55e8 100644 --- a/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp +++ b/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp @@ -67,10 +67,22 @@ GeometryBuffersBuilder::Buffers TGaussianSplatBuffersBuilder::ConstructBuffers() const auto& points = geometry_.GetPointPositions(); const size_t n_vertices = points.GetLength(); + int sh_degree = geometry_.GaussianSplatGetSHOrder(); + if (sh_degree > 2) { + // If the degree of SH coeffs is higher than 2, we treate it as degree 2. + sh_degree = 2; + } + + int f_rest_coeffs_count = sh_degree * (sh_degree + 2) * 3; + int f_rest_buffer_count = (f_rest_coeffs_count + 3) / 4; + + int base_buffer_count = 4; + int all_buffer_count = base_buffer_count + f_rest_buffer_count; + // we use POSITION for positions, COLOR for scale, TANGENTS for rot // CUSTOM0 for f_dc and opacity, CUSTOM1-CUSTOM6 for f_rest. - VertexBuffer* vbuf = VertexBuffer::Builder() - .bufferCount(10) + VertexBuffer::Builder buffer_builder = VertexBuffer::Builder() + .bufferCount(all_buffer_count) .vertexCount(uint32_t(n_vertices)) .attribute(VertexAttribute::POSITION, 0, VertexBuffer::AttributeType::FLOAT3) @@ -79,20 +91,19 @@ GeometryBuffersBuilder::Buffers TGaussianSplatBuffersBuilder::ConstructBuffers() .attribute(VertexAttribute::TANGENTS, 2, VertexBuffer::AttributeType::FLOAT4) .attribute(VertexAttribute::CUSTOM0, 3, - VertexBuffer::AttributeType::FLOAT4) - .attribute(VertexAttribute::CUSTOM1, 4, - VertexBuffer::AttributeType::FLOAT4) - .attribute(VertexAttribute::CUSTOM2, 5, - VertexBuffer::AttributeType::FLOAT4) - .attribute(VertexAttribute::CUSTOM3, 6, - VertexBuffer::AttributeType::FLOAT4) - .attribute(VertexAttribute::CUSTOM4, 7, - VertexBuffer::AttributeType::FLOAT4) - .attribute(VertexAttribute::CUSTOM5, 8, - VertexBuffer::AttributeType::FLOAT4) - .attribute(VertexAttribute::CUSTOM6, 9, - VertexBuffer::AttributeType::FLOAT4) - .build(engine); + VertexBuffer::AttributeType::FLOAT4); + if (sh_degree >= 1) { + buffer_builder.attribute(VertexAttribute::CUSTOM1, 4, VertexBuffer::AttributeType::FLOAT4); + buffer_builder.attribute(VertexAttribute::CUSTOM2, 5, VertexBuffer::AttributeType::FLOAT4); + buffer_builder.attribute(VertexAttribute::CUSTOM3, 6, VertexBuffer::AttributeType::FLOAT4); + } + if (sh_degree == 2) { + buffer_builder.attribute(VertexAttribute::CUSTOM4, 7, VertexBuffer::AttributeType::FLOAT4); + buffer_builder.attribute(VertexAttribute::CUSTOM5, 8, VertexBuffer::AttributeType::FLOAT4); + buffer_builder.attribute(VertexAttribute::CUSTOM6, 9, VertexBuffer::AttributeType::FLOAT4); + } + + VertexBuffer* vbuf = buffer_builder.build(engine); VertexBufferHandle vb_handle; if (vbuf) { @@ -144,17 +155,22 @@ GeometryBuffersBuilder::Buffers TGaussianSplatBuffersBuilder::ConstructBuffers() GeometryBuffersBuilder::DeallocateBuffer); vbuf->setBufferAt(engine, 3, std::move(color_descriptor)); - const size_t f_rest_array_size = n_vertices * 4 * sizeof(float); - const size_t custom_buffer_num = 6; + int data_count_in_one_buffer = 4; + const size_t f_rest_array_size = n_vertices * data_count_in_one_buffer * sizeof(float); const size_t custom_buffer_start_index = 4; - for(size_t i = 0; i < custom_buffer_num; i++) { + for(int i = 0; i < f_rest_buffer_count; i++) { float* f_rest_array = static_cast(malloc(f_rest_array_size)); - if (geometry_.HasPointAttr("f_rest")) { - float* f_rest_src = geometry_.GetPointAttr("f_rest").GetDataPtr(); - std::memcpy(f_rest_array, f_rest_src + i * n_vertices * 4, f_rest_array_size); - } else { - std::memset(f_rest_array, 0, f_rest_array_size); + std::memset(f_rest_array, 0, f_rest_array_size); + + size_t copy_data_size = f_rest_array_size; + if (i == f_rest_buffer_count -1) { + int remaining_count_in_last_iter = data_count_in_one_buffer + (f_rest_coeffs_count + - f_rest_buffer_count * data_count_in_one_buffer); + copy_data_size = n_vertices * remaining_count_in_last_iter * sizeof(float); } + + float* f_rest_src = geometry_.GetPointAttr("f_rest").GetDataPtr(); + std::memcpy(f_rest_array, f_rest_src + i * n_vertices * data_count_in_one_buffer, copy_data_size); VertexBuffer::BufferDescriptor f_rest_descriptor( f_rest_array, f_rest_array_size, GeometryBuffersBuilder::DeallocateBuffer); vbuf->setBufferAt(engine, custom_buffer_start_index + i, std::move(f_rest_descriptor)); From c67ecc068b759b9c16d9de1125269e88a41bbf4e Mon Sep 17 00:00:00 2001 From: Zhang Yang Date: Thu, 13 Mar 2025 16:21:35 +0800 Subject: [PATCH 14/28] add comment --- .../visualization/rendering/filament/GaussianSplatBuffers.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp b/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp index 45fc00a55e8..1608da2c744 100644 --- a/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp +++ b/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp @@ -169,6 +169,7 @@ GeometryBuffersBuilder::Buffers TGaussianSplatBuffersBuilder::ConstructBuffers() copy_data_size = n_vertices * remaining_count_in_last_iter * sizeof(float); } + // f_rest_buffer_count is not 0 means that can get f_rest data here. float* f_rest_src = geometry_.GetPointAttr("f_rest").GetDataPtr(); std::memcpy(f_rest_array, f_rest_src + i * n_vertices * data_count_in_one_buffer, copy_data_size); VertexBuffer::BufferDescriptor f_rest_descriptor( From 9eb09b68043cf8cab311b0dbcc6ff26729b09eed Mon Sep 17 00:00:00 2001 From: Zhang Yang Date: Fri, 14 Mar 2025 10:39:14 +0800 Subject: [PATCH 15/28] simplify GaussianSplatGetSHOrder format --- cpp/open3d/t/geometry/PointCloud.cpp | 58 +++++----------------------- 1 file changed, 10 insertions(+), 48 deletions(-) diff --git a/cpp/open3d/t/geometry/PointCloud.cpp b/cpp/open3d/t/geometry/PointCloud.cpp index 7ae5ab4de7f..fb80439e5cf 100644 --- a/cpp/open3d/t/geometry/PointCloud.cpp +++ b/cpp/open3d/t/geometry/PointCloud.cpp @@ -1384,62 +1384,24 @@ int PointCloud::GaussianSplatGetSHOrder() const { return 0; } else { const core::Tensor & f_rest = GetPointAttr("f_rest"); - auto shape = f_rest.GetShape(); - size_t num_dims = shape.size(); + // core::AssertTensorShape(f_rest, {core::None, core::None, 3}); + auto Nc = f_rest.GetShape(1); - int dim_of_degree = shape[1]; + // comment the following part later + size_t num_dims = f_rest.GetShape().size(); if (num_dims == 2) { - if (dim_of_degree % 3 != 0) { + if (Nc % 3 != 0) { utility::LogError("SH coeffs are in shape [N, Nc], but the dimension of Nc cannot be divided by 3"); } - dim_of_degree = dim_of_degree / 3; + Nc = Nc / 3; } - std::optional degree = findPositiveIntegerSolution(dim_of_degree); - if (degree.has_value()) { - if (degree.value() > 2) { - utility::LogWarning("The SH coeffs degree higher than 2 is not supported for now."); - } - return degree.value(); - } else { - utility::LogError("The number of SH coeffs is not a valid value."); + auto deg = static_cast(sqrt(Nc+1)); + if (deg * deg != Nc+1) { + utility::LogError("f_rest has incomplete Spherical Harmonics coefficients ({}), expected 0, 3, 8 or 15.", Nc); } - return -1; - } -} - -std::optional PointCloud::findPositiveIntegerSolution(int x) const { - // Coefficients of the quadratic equation n^2 + 2n - x = 0 - int a = 1; - int b = 2; - int c = -x; - - // Calculate the discriminant - int discriminant = b * b - 4 * a * c; - - // Check if the discriminant is non-negative and a perfect square - if (discriminant < 0) { - return std::nullopt; + return deg - 1; } - - int sqrt_discriminant = static_cast(std::sqrt(discriminant)); - if (sqrt_discriminant * sqrt_discriminant != discriminant) { - return std::nullopt; - } - - // Calculate the two possible solutions for n - int n1 = (-b + sqrt_discriminant) / (2 * a); - int n2 = (-b - sqrt_discriminant) / (2 * a); - - // Check if either solution is a valid positive integer - // There is definitely a positive result because x is positive. - if (n1 > 0 && n1 * (n1 + 2) == x) { - return n1; - } else if (n2 > 0 && n2 * (n2 + 2) == x) { - return n2; - } - - return std::nullopt; } } // namespace geometry } // namespace t From 665b72b546dcf3ee186f830b9cbaf017860a76f8 Mon Sep 17 00:00:00 2001 From: Zhang Yang Date: Fri, 14 Mar 2025 11:12:51 +0800 Subject: [PATCH 16/28] clang format --- cpp/open3d/t/geometry/PointCloud.cpp | 20 ++-- cpp/open3d/t/geometry/PointCloud.h | 11 +- .../FilamentGeometryBuffersBuilder.cpp | 6 +- .../filament/FilamentGeometryBuffersBuilder.h | 27 +++-- .../filament/FilamentResourceManager.cpp | 11 +- .../filament/GaussianSplatBuffers.cpp | 112 +++++++++++------- .../rendering/filament/PointCloudBuffers.cpp | 3 +- 7 files changed, 115 insertions(+), 75 deletions(-) diff --git a/cpp/open3d/t/geometry/PointCloud.cpp b/cpp/open3d/t/geometry/PointCloud.cpp index fb80439e5cf..3bc92c6d1ab 100644 --- a/cpp/open3d/t/geometry/PointCloud.cpp +++ b/cpp/open3d/t/geometry/PointCloud.cpp @@ -1370,8 +1370,9 @@ core::Tensor PointCloud::ComputeMetrics(const PointCloud &pcd2, } bool PointCloud::IsGaussianSplat() const { - std::vector keys_to_check = {"f_dc", "opacity", "rot", "scale"}; - for (const auto& key : keys_to_check) { + std::vector keys_to_check = {"f_dc", "opacity", "rot", + "scale"}; + for (const auto &key : keys_to_check) { if (point_attr_.find(key) == point_attr_.end()) { return false; } @@ -1383,7 +1384,7 @@ int PointCloud::GaussianSplatGetSHOrder() const { if (point_attr_.find("f_rest") == point_attr_.end()) { return 0; } else { - const core::Tensor & f_rest = GetPointAttr("f_rest"); + const core::Tensor &f_rest = GetPointAttr("f_rest"); // core::AssertTensorShape(f_rest, {core::None, core::None, 3}); auto Nc = f_rest.GetShape(1); @@ -1391,14 +1392,19 @@ int PointCloud::GaussianSplatGetSHOrder() const { size_t num_dims = f_rest.GetShape().size(); if (num_dims == 2) { if (Nc % 3 != 0) { - utility::LogError("SH coeffs are in shape [N, Nc], but the dimension of Nc cannot be divided by 3"); + utility::LogError( + "SH coeffs are in shape [N, Nc], but the dimension of " + "Nc cannot be divided by 3"); } Nc = Nc / 3; } - auto deg = static_cast(sqrt(Nc+1)); - if (deg * deg != Nc+1) { - utility::LogError("f_rest has incomplete Spherical Harmonics coefficients ({}), expected 0, 3, 8 or 15.", Nc); + auto deg = static_cast(sqrt(Nc + 1)); + if (deg * deg != Nc + 1) { + utility::LogError( + "f_rest has incomplete Spherical Harmonics coefficients " + "({}), expected 0, 3, 8 or 15.", + Nc); } return deg - 1; } diff --git a/cpp/open3d/t/geometry/PointCloud.h b/cpp/open3d/t/geometry/PointCloud.h index 35861482294..a87de785506 100644 --- a/cpp/open3d/t/geometry/PointCloud.h +++ b/cpp/open3d/t/geometry/PointCloud.h @@ -740,13 +740,16 @@ class PointCloud : public Geometry, public DrawableGeometry { /// \brief Checks if the point cloud is formatted for Gaussian Splatting. /// - /// This function verifies that the point attribute container contains all the required keys - /// ("f_dc", "opacity", "rot", and "scale") necessary for Gaussian Splatting. If any of these keys - /// are missing, the function returns \c false; otherwise, it returns \c true. - /// \return \c true if all required attributes are present, \c false otherwise. + /// This function verifies that the point attribute container contains all + /// the required keys + /// ("f_dc", "opacity", "rot", and "scale") necessary for Gaussian + /// Splatting. If any of these keys are missing, the function returns \c + /// false; otherwise, it returns \c true. \return \c true if all required + /// attributes are present, \c false otherwise. bool IsGaussianSplat() const; int GaussianSplatGetSHOrder() const; + protected: core::Device device_ = core::Device("CPU:0"); TensorMap point_attr_; diff --git a/cpp/open3d/visualization/rendering/filament/FilamentGeometryBuffersBuilder.cpp b/cpp/open3d/visualization/rendering/filament/FilamentGeometryBuffersBuilder.cpp index aa5d3dfa212..1a09943c699 100644 --- a/cpp/open3d/visualization/rendering/filament/FilamentGeometryBuffersBuilder.cpp +++ b/cpp/open3d/visualization/rendering/filament/FilamentGeometryBuffersBuilder.cpp @@ -251,9 +251,11 @@ std::unique_ptr GeometryBuffersBuilder::GetBuilder( switch (geometry.GetGeometryType()) { case GT::PointCloud: { - const t::geometry::PointCloud& pointcloud = static_cast(geometry); + const t::geometry::PointCloud& pointcloud = + static_cast(geometry); if (pointcloud.IsGaussianSplat()) { - return std::make_unique(pointcloud); + return std::make_unique( + pointcloud); } else { return std::make_unique(pointcloud); } diff --git a/cpp/open3d/visualization/rendering/filament/FilamentGeometryBuffersBuilder.h b/cpp/open3d/visualization/rendering/filament/FilamentGeometryBuffersBuilder.h index e8130697853..40bbd32d349 100644 --- a/cpp/open3d/visualization/rendering/filament/FilamentGeometryBuffersBuilder.h +++ b/cpp/open3d/visualization/rendering/filament/FilamentGeometryBuffersBuilder.h @@ -180,19 +180,22 @@ class TGaussianSplatBuffersBuilder : public TPointCloudBuffersBuilder { public: /// \brief Constructs a TGaussianSplatBuffersBuilder object. /// - /// Initializes the Gaussian Splat buffers from the provided \p geometry and ensures that - /// all necessary attributes are present and correctly formatted. If the geometry is not a - /// Gaussian Splat, a warning is issued. Additionally, attributes like "f_dc", "opacity", - /// "rot", "scale", and "f_rest" are checked for their data type, and converted to Float32 - /// if they are not already in that format. - explicit TGaussianSplatBuffersBuilder(const t::geometry::PointCloud& geometry); + /// Initializes the Gaussian Splat buffers from the provided \p geometry and + /// ensures that all necessary attributes are present and correctly + /// formatted. If the geometry is not a Gaussian Splat, a warning is issued. + /// Additionally, attributes like "f_dc", "opacity", "rot", "scale", and + /// "f_rest" are checked for their data type, and converted to Float32 if + /// they are not already in that format. + explicit TGaussianSplatBuffersBuilder( + const t::geometry::PointCloud& geometry); /// \brief Constructs vertex and index buffers for Gaussian Splat rendering. /// - /// This function creates and configures GPU buffers to represent a Gaussian Splat - /// point cloud. It extracts attributes like positions, colors, rotation, scale, - /// and spherical harmonics coefficients from the provided \ref geometry_ and - /// organizes them into separate vertex buffer attributes. + /// This function creates and configures GPU buffers to represent a Gaussian + /// Splat point cloud. It extracts attributes like positions, colors, + /// rotation, scale, and spherical harmonics coefficients from the provided + /// \ref geometry_ and organizes them into separate vertex buffer + /// attributes. /// /// The vertex buffer contains the following attributes: /// - POSITION: Vertex positions (FLOAT3) @@ -201,8 +204,8 @@ class TGaussianSplatBuffersBuilder : public TPointCloudBuffersBuilder { /// - CUSTOM0: Scale (FLOAT4) /// - CUSTOM1 to CUSTOM6: SH coefficients (FLOAT4) /// - /// Each attribute is checked and converted to the expected data type if necessary, - /// and missing attributes are initialized with default values. + /// Each attribute is checked and converted to the expected data type if + /// necessary, and missing attributes are initialized with default values. Buffers ConstructBuffers() override; }; diff --git a/cpp/open3d/visualization/rendering/filament/FilamentResourceManager.cpp b/cpp/open3d/visualization/rendering/filament/FilamentResourceManager.cpp index a0504b05f58..dd10e441f59 100644 --- a/cpp/open3d/visualization/rendering/filament/FilamentResourceManager.cpp +++ b/cpp/open3d/visualization/rendering/filament/FilamentResourceManager.cpp @@ -1022,7 +1022,7 @@ void FilamentResourceManager::LoadDefaults() { const auto gaussian_path = resource_root + "/gaussianSplat.filamat"; auto gaussian_mat = LoadMaterialFromFile(gaussian_path, engine_); gaussian_mat->setDefaultParameter("baseColor", filament::RgbType::sRGB, - default_color); + default_color); gaussian_mat->setDefaultParameter("baseRoughness", 0.7f); gaussian_mat->setDefaultParameter("reflectance", 0.5f); gaussian_mat->setDefaultParameter("baseMetallic", 0.f); @@ -1031,11 +1031,14 @@ void FilamentResourceManager::LoadDefaults() { gaussian_mat->setDefaultParameter("anisotropy", 0.f); gaussian_mat->setDefaultParameter("pointSize", 3.f); gaussian_mat->setDefaultParameter("albedo", texture, default_sampler); - gaussian_mat->setDefaultParameter("ao_rough_metalMap", texture, default_sampler); + gaussian_mat->setDefaultParameter("ao_rough_metalMap", texture, + default_sampler); gaussian_mat->setDefaultParameter("normalMap", normal_map, default_sampler); - gaussian_mat->setDefaultParameter("reflectanceMap", texture, default_sampler); + gaussian_mat->setDefaultParameter("reflectanceMap", texture, + default_sampler); - gaussian_mat->setDefaultParameter("anisotropyMap", texture, default_sampler); + gaussian_mat->setDefaultParameter("anisotropyMap", texture, + default_sampler); materials_[kGaussianSplatShader] = BoxResource(gaussian_mat, engine_); const auto lit_trans_path = diff --git a/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp b/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp index 1608da2c744..854e0711772 100644 --- a/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp +++ b/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp @@ -42,25 +42,29 @@ TGaussianSplatBuffersBuilder::TGaussianSplatBuffersBuilder( : TPointCloudBuffersBuilder(geometry) { if (!geometry.IsGaussianSplat()) { utility::LogWarning( - "TGaussianSplatBuffers is constructed for a geometry that is not GaussianSplat." - ); + "TGaussianSplatBuffers is constructed for a geometry that is " + "not GaussianSplat."); } - std::vector check_list = {"f_dc", "opacity", "rot", "scale", "f_rest"}; + std::vector check_list = {"f_dc", "opacity", "rot", "scale", + "f_rest"}; for (const auto& check_item : check_list) { if (geometry_.HasPointAttr(check_item) && geometry_.GetPointAttr(check_item).GetDtype() != core::Float32) { auto check_item_instance = geometry_.GetPointAttr(check_item); utility::LogWarning( - "Tensor gaussian splat {} must have DType of Float32 not {}. " + "Tensor gaussian splat {} must have DType of Float32 not " + "{}. " "Converting.", check_item, check_item_instance.GetDtype().ToString()); - geometry_.GetPointAttr(check_item) = check_item_instance.To(core::Float32); + geometry_.GetPointAttr(check_item) = + check_item_instance.To(core::Float32); } } } -GeometryBuffersBuilder::Buffers TGaussianSplatBuffersBuilder::ConstructBuffers() { +GeometryBuffersBuilder::Buffers +TGaussianSplatBuffersBuilder::ConstructBuffers() { auto& engine = EngineInstance::GetInstance(); auto& resource_mgr = EngineInstance::GetResourceManager(); @@ -69,8 +73,9 @@ GeometryBuffersBuilder::Buffers TGaussianSplatBuffersBuilder::ConstructBuffers() int sh_degree = geometry_.GaussianSplatGetSHOrder(); if (sh_degree > 2) { - // If the degree of SH coeffs is higher than 2, we treate it as degree 2. - sh_degree = 2; + // If the degree of SH coeffs is higher than 2, we treate it as + // degree 2. + sh_degree = 2; } int f_rest_coeffs_count = sh_degree * (sh_degree + 2) * 3; @@ -79,28 +84,35 @@ GeometryBuffersBuilder::Buffers TGaussianSplatBuffersBuilder::ConstructBuffers() int base_buffer_count = 4; int all_buffer_count = base_buffer_count + f_rest_buffer_count; - // we use POSITION for positions, COLOR for scale, TANGENTS for rot - // CUSTOM0 for f_dc and opacity, CUSTOM1-CUSTOM6 for f_rest. - VertexBuffer::Builder buffer_builder = VertexBuffer::Builder() - .bufferCount(all_buffer_count) - .vertexCount(uint32_t(n_vertices)) - .attribute(VertexAttribute::POSITION, 0, - VertexBuffer::AttributeType::FLOAT3) - .attribute(VertexAttribute::COLOR, 1, - VertexBuffer::AttributeType::FLOAT4) - .attribute(VertexAttribute::TANGENTS, 2, - VertexBuffer::AttributeType::FLOAT4) - .attribute(VertexAttribute::CUSTOM0, 3, - VertexBuffer::AttributeType::FLOAT4); + // we use POSITION for positions, COLOR for scale, TANGENTS for rot + // CUSTOM0 for f_dc and opacity, CUSTOM1-CUSTOM6 for f_rest. + VertexBuffer::Builder buffer_builder = + VertexBuffer::Builder() + .bufferCount(all_buffer_count) + .vertexCount(uint32_t(n_vertices)) + .attribute(VertexAttribute::POSITION, 0, + VertexBuffer::AttributeType::FLOAT3) + .attribute(VertexAttribute::COLOR, 1, + VertexBuffer::AttributeType::FLOAT4) + .attribute(VertexAttribute::TANGENTS, 2, + VertexBuffer::AttributeType::FLOAT4) + .attribute(VertexAttribute::CUSTOM0, 3, + VertexBuffer::AttributeType::FLOAT4); if (sh_degree >= 1) { - buffer_builder.attribute(VertexAttribute::CUSTOM1, 4, VertexBuffer::AttributeType::FLOAT4); - buffer_builder.attribute(VertexAttribute::CUSTOM2, 5, VertexBuffer::AttributeType::FLOAT4); - buffer_builder.attribute(VertexAttribute::CUSTOM3, 6, VertexBuffer::AttributeType::FLOAT4); + buffer_builder.attribute(VertexAttribute::CUSTOM1, 4, + VertexBuffer::AttributeType::FLOAT4); + buffer_builder.attribute(VertexAttribute::CUSTOM2, 5, + VertexBuffer::AttributeType::FLOAT4); + buffer_builder.attribute(VertexAttribute::CUSTOM3, 6, + VertexBuffer::AttributeType::FLOAT4); } if (sh_degree == 2) { - buffer_builder.attribute(VertexAttribute::CUSTOM4, 7, VertexBuffer::AttributeType::FLOAT4); - buffer_builder.attribute(VertexAttribute::CUSTOM5, 8, VertexBuffer::AttributeType::FLOAT4); - buffer_builder.attribute(VertexAttribute::CUSTOM6, 9, VertexBuffer::AttributeType::FLOAT4); + buffer_builder.attribute(VertexAttribute::CUSTOM4, 7, + VertexBuffer::AttributeType::FLOAT4); + buffer_builder.attribute(VertexAttribute::CUSTOM5, 8, + VertexBuffer::AttributeType::FLOAT4); + buffer_builder.attribute(VertexAttribute::CUSTOM6, 9, + VertexBuffer::AttributeType::FLOAT4); } VertexBuffer* vbuf = buffer_builder.build(engine); @@ -125,24 +137,25 @@ GeometryBuffersBuilder::Buffers TGaussianSplatBuffersBuilder::ConstructBuffers() std::memset(scale_array, 0, scale_array_size); float* scale_src = geometry_.GetPointAttr("scale").GetDataPtr(); for (size_t i = 0; i < n_vertices; i++) { - std::memcpy(scale_array + i * 4, scale_src + i * 3, 3*sizeof(float)); + std::memcpy(scale_array + i * 4, scale_src + i * 3, 3 * sizeof(float)); } VertexBuffer::BufferDescriptor scale_descriptor( - scale_array, scale_array_size, GeometryBuffersBuilder::DeallocateBuffer); + scale_array, scale_array_size, + GeometryBuffersBuilder::DeallocateBuffer); vbuf->setBufferAt(engine, 1, std::move(scale_descriptor)); const size_t rot_array_size = n_vertices * 4 * sizeof(float); float* rot_array = static_cast(malloc(rot_array_size)); std::memset(rot_array, 0, rot_array_size); std::memcpy(rot_array, geometry_.GetPointAttr("rot").GetDataPtr(), - rot_array_size); + rot_array_size); VertexBuffer::BufferDescriptor rot_descriptor( - rot_array, rot_array_size, - GeometryBuffersBuilder::DeallocateBuffer); + rot_array, rot_array_size, + GeometryBuffersBuilder::DeallocateBuffer); vbuf->setBufferAt(engine, 2, std::move(rot_descriptor)); const size_t color_array_size = n_vertices * 4 * sizeof(float); - float* color_array = static_cast(malloc(color_array_size)); + float* color_array = static_cast(malloc(color_array_size)); std::memset(color_array, 0, color_array_size); float* f_dc_ptr = geometry_.GetPointAttr("f_dc").GetDataPtr(); float* opacity_ptr = geometry_.GetPointAttr("opacity").GetDataPtr(); @@ -151,30 +164,39 @@ GeometryBuffersBuilder::Buffers TGaussianSplatBuffersBuilder::ConstructBuffers() std::memcpy(color_array + i * 4 + 3, opacity_ptr + i, sizeof(float)); } VertexBuffer::BufferDescriptor color_descriptor( - color_array, color_array_size, - GeometryBuffersBuilder::DeallocateBuffer); + color_array, color_array_size, + GeometryBuffersBuilder::DeallocateBuffer); vbuf->setBufferAt(engine, 3, std::move(color_descriptor)); int data_count_in_one_buffer = 4; - const size_t f_rest_array_size = n_vertices * data_count_in_one_buffer * sizeof(float); + const size_t f_rest_array_size = + n_vertices * data_count_in_one_buffer * sizeof(float); const size_t custom_buffer_start_index = 4; - for(int i = 0; i < f_rest_buffer_count; i++) { + for (int i = 0; i < f_rest_buffer_count; i++) { float* f_rest_array = static_cast(malloc(f_rest_array_size)); std::memset(f_rest_array, 0, f_rest_array_size); size_t copy_data_size = f_rest_array_size; - if (i == f_rest_buffer_count -1) { - int remaining_count_in_last_iter = data_count_in_one_buffer + (f_rest_coeffs_count - - f_rest_buffer_count * data_count_in_one_buffer); - copy_data_size = n_vertices * remaining_count_in_last_iter * sizeof(float); + if (i == f_rest_buffer_count - 1) { + int remaining_count_in_last_iter = + data_count_in_one_buffer + + (f_rest_coeffs_count - + f_rest_buffer_count * data_count_in_one_buffer); + copy_data_size = + n_vertices * remaining_count_in_last_iter * sizeof(float); } // f_rest_buffer_count is not 0 means that can get f_rest data here. - float* f_rest_src = geometry_.GetPointAttr("f_rest").GetDataPtr(); - std::memcpy(f_rest_array, f_rest_src + i * n_vertices * data_count_in_one_buffer, copy_data_size); + float* f_rest_src = + geometry_.GetPointAttr("f_rest").GetDataPtr(); + std::memcpy(f_rest_array, + f_rest_src + i * n_vertices * data_count_in_one_buffer, + copy_data_size); VertexBuffer::BufferDescriptor f_rest_descriptor( - f_rest_array, f_rest_array_size, GeometryBuffersBuilder::DeallocateBuffer); - vbuf->setBufferAt(engine, custom_buffer_start_index + i, std::move(f_rest_descriptor)); + f_rest_array, f_rest_array_size, + GeometryBuffersBuilder::DeallocateBuffer); + vbuf->setBufferAt(engine, custom_buffer_start_index + i, + std::move(f_rest_descriptor)); } auto ib_handle = CreateIndexBuffer(n_vertices); diff --git a/cpp/open3d/visualization/rendering/filament/PointCloudBuffers.cpp b/cpp/open3d/visualization/rendering/filament/PointCloudBuffers.cpp index d14ac5394bb..f88db292b1f 100644 --- a/cpp/open3d/visualization/rendering/filament/PointCloudBuffers.cpp +++ b/cpp/open3d/visualization/rendering/filament/PointCloudBuffers.cpp @@ -434,7 +434,8 @@ GeometryBuffersBuilder::Buffers TPointCloudBuffersBuilder::ConstructBuffers() { const size_t uv_array_size = n_vertices * 2 * sizeof(float); float* uv_array = static_cast(malloc(uv_array_size)); if (geometry_.HasPointAttr("uv")) { - const float* uv_src = geometry_.GetPointAttr("uv").GetDataPtr(); + const float* uv_src = + geometry_.GetPointAttr("uv").GetDataPtr(); memcpy(uv_array, uv_src, uv_array_size); } else if (geometry_.HasPointAttr("__visualization_scalar")) { // Update in FilamentScene::UpdateGeometry(), too. From 4250a9c6098b01938cc0e6d016f7b50bfd86243e Mon Sep 17 00:00:00 2001 From: Zhang Yang Date: Fri, 14 Mar 2025 11:29:19 +0800 Subject: [PATCH 17/28] update logic to calculate f rest buffer count --- .../rendering/filament/GaussianSplatBuffers.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp b/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp index 854e0711772..434f2790f5c 100644 --- a/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp +++ b/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp @@ -79,7 +79,9 @@ TGaussianSplatBuffersBuilder::ConstructBuffers() { } int f_rest_coeffs_count = sh_degree * (sh_degree + 2) * 3; - int f_rest_buffer_count = (f_rest_coeffs_count + 3) / 4; + int f_rest_buffer_count = (f_rest_coeffs_count % 4 == 0) + ? (f_rest_coeffs_count / 4) + : std::ceil(f_rest_coeffs_count / 4.0); int base_buffer_count = 4; int all_buffer_count = base_buffer_count + f_rest_buffer_count; @@ -174,7 +176,6 @@ TGaussianSplatBuffersBuilder::ConstructBuffers() { const size_t custom_buffer_start_index = 4; for (int i = 0; i < f_rest_buffer_count; i++) { float* f_rest_array = static_cast(malloc(f_rest_array_size)); - std::memset(f_rest_array, 0, f_rest_array_size); size_t copy_data_size = f_rest_array_size; if (i == f_rest_buffer_count - 1) { From 1d580c31cfdb1c8df6b461ea5ebf5362b9185efb Mon Sep 17 00:00:00 2001 From: Zhang Yang Date: Fri, 14 Mar 2025 15:31:31 +0800 Subject: [PATCH 18/28] remove useless memset --- .../visualization/rendering/filament/GaussianSplatBuffers.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp b/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp index 434f2790f5c..7c8a7aa9d2d 100644 --- a/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp +++ b/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp @@ -148,7 +148,6 @@ TGaussianSplatBuffersBuilder::ConstructBuffers() { const size_t rot_array_size = n_vertices * 4 * sizeof(float); float* rot_array = static_cast(malloc(rot_array_size)); - std::memset(rot_array, 0, rot_array_size); std::memcpy(rot_array, geometry_.GetPointAttr("rot").GetDataPtr(), rot_array_size); VertexBuffer::BufferDescriptor rot_descriptor( @@ -158,7 +157,6 @@ TGaussianSplatBuffersBuilder::ConstructBuffers() { const size_t color_array_size = n_vertices * 4 * sizeof(float); float* color_array = static_cast(malloc(color_array_size)); - std::memset(color_array, 0, color_array_size); float* f_dc_ptr = geometry_.GetPointAttr("f_dc").GetDataPtr(); float* opacity_ptr = geometry_.GetPointAttr("opacity").GetDataPtr(); for (size_t i = 0; i < n_vertices; i++) { @@ -185,6 +183,8 @@ TGaussianSplatBuffersBuilder::ConstructBuffers() { f_rest_buffer_count * data_count_in_one_buffer); copy_data_size = n_vertices * remaining_count_in_last_iter * sizeof(float); + std::memset(f_rest_src + i * n_vertices * data_count_in_one_buffer, + 0, f_rest_array_size); } // f_rest_buffer_count is not 0 means that can get f_rest data here. From ef2c6150ddfa5611aedaa0f730a39a0a7eec9545 Mon Sep 17 00:00:00 2001 From: Zhang Yang Date: Fri, 14 Mar 2025 17:49:11 +0800 Subject: [PATCH 19/28] bug fix --- .../rendering/filament/GaussianSplatBuffers.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp b/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp index 7c8a7aa9d2d..6f2f8ddd52d 100644 --- a/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp +++ b/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp @@ -172,6 +172,11 @@ TGaussianSplatBuffersBuilder::ConstructBuffers() { const size_t f_rest_array_size = n_vertices * data_count_in_one_buffer * sizeof(float); const size_t custom_buffer_start_index = 4; + + float* f_rest_src = + (f_rest_buffer_count > 0) + ? geometry_.GetPointAttr("f_rest").GetDataPtr() + : nullptr; for (int i = 0; i < f_rest_buffer_count; i++) { float* f_rest_array = static_cast(malloc(f_rest_array_size)); @@ -187,9 +192,6 @@ TGaussianSplatBuffersBuilder::ConstructBuffers() { 0, f_rest_array_size); } - // f_rest_buffer_count is not 0 means that can get f_rest data here. - float* f_rest_src = - geometry_.GetPointAttr("f_rest").GetDataPtr(); std::memcpy(f_rest_array, f_rest_src + i * n_vertices * data_count_in_one_buffer, copy_data_size); From 86d47270721c6aea6d461ffe72bdd5835582139d Mon Sep 17 00:00:00 2001 From: Zhang Yang Date: Mon, 17 Mar 2025 11:30:57 +0800 Subject: [PATCH 20/28] check sh degree in GaussianSplatGetSHOrder() --- cpp/open3d/t/geometry/PointCloud.cpp | 9 +++++++++ .../rendering/filament/GaussianSplatBuffers.cpp | 5 ----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/cpp/open3d/t/geometry/PointCloud.cpp b/cpp/open3d/t/geometry/PointCloud.cpp index 3bc92c6d1ab..652bf60182b 100644 --- a/cpp/open3d/t/geometry/PointCloud.cpp +++ b/cpp/open3d/t/geometry/PointCloud.cpp @@ -1406,6 +1406,15 @@ int PointCloud::GaussianSplatGetSHOrder() const { "({}), expected 0, 3, 8 or 15.", Nc); } + + deg = deg - 1; + if (deg > 2) { + utility::LogWarning( + "SH degree higher than 2 is not supported. The SH degree " + "of in input data is {}, we set it as 2.", + deg); + } + return deg - 1; } } diff --git a/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp b/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp index 6f2f8ddd52d..0397fe37394 100644 --- a/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp +++ b/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp @@ -72,11 +72,6 @@ TGaussianSplatBuffersBuilder::ConstructBuffers() { const size_t n_vertices = points.GetLength(); int sh_degree = geometry_.GaussianSplatGetSHOrder(); - if (sh_degree > 2) { - // If the degree of SH coeffs is higher than 2, we treate it as - // degree 2. - sh_degree = 2; - } int f_rest_coeffs_count = sh_degree * (sh_degree + 2) * 3; int f_rest_buffer_count = (f_rest_coeffs_count % 4 == 0) From 9d3d6393f11ec472ce99959652c756440377a99e Mon Sep 17 00:00:00 2001 From: Zhang Yang Date: Tue, 18 Mar 2025 10:13:48 +0800 Subject: [PATCH 21/28] bug fix --- cpp/open3d/t/geometry/PointCloud.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/open3d/t/geometry/PointCloud.cpp b/cpp/open3d/t/geometry/PointCloud.cpp index 652bf60182b..7bbcac40eb8 100644 --- a/cpp/open3d/t/geometry/PointCloud.cpp +++ b/cpp/open3d/t/geometry/PointCloud.cpp @@ -1415,7 +1415,7 @@ int PointCloud::GaussianSplatGetSHOrder() const { deg); } - return deg - 1; + return deg; } } } // namespace geometry From ce708dcb26d6b891bf6f31b171c433478ae1ccfe Mon Sep 17 00:00:00 2001 From: Zhang Yang Date: Tue, 18 Mar 2025 10:31:09 +0800 Subject: [PATCH 22/28] bug fix --- cpp/open3d/t/geometry/PointCloud.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/open3d/t/geometry/PointCloud.cpp b/cpp/open3d/t/geometry/PointCloud.cpp index 7bbcac40eb8..6988376ada3 100644 --- a/cpp/open3d/t/geometry/PointCloud.cpp +++ b/cpp/open3d/t/geometry/PointCloud.cpp @@ -1413,6 +1413,7 @@ int PointCloud::GaussianSplatGetSHOrder() const { "SH degree higher than 2 is not supported. The SH degree " "of in input data is {}, we set it as 2.", deg); + deg = 2; } return deg; From 897601da8c0ef552d5ac8d44293013f9ca21f665 Mon Sep 17 00:00:00 2001 From: Zhang Yang Date: Tue, 18 Mar 2025 14:02:08 +0800 Subject: [PATCH 23/28] fix bug --- .../filament/GaussianSplatBuffers.cpp | 50 +++++++++++-------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp b/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp index 0397fe37394..181a4184b57 100644 --- a/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp +++ b/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp @@ -78,10 +78,10 @@ TGaussianSplatBuffersBuilder::ConstructBuffers() { ? (f_rest_coeffs_count / 4) : std::ceil(f_rest_coeffs_count / 4.0); - int base_buffer_count = 4; + int base_buffer_count = 5; int all_buffer_count = base_buffer_count + f_rest_buffer_count; - // we use POSITION for positions, COLOR for scale, TANGENTS for rot + // we use POSITION for positions, COLOR for scale, CUSTOM7 for rot // CUSTOM0 for f_dc and opacity, CUSTOM1-CUSTOM6 for f_rest. VertexBuffer::Builder buffer_builder = VertexBuffer::Builder() @@ -92,23 +92,25 @@ TGaussianSplatBuffersBuilder::ConstructBuffers() { .attribute(VertexAttribute::COLOR, 1, VertexBuffer::AttributeType::FLOAT4) .attribute(VertexAttribute::TANGENTS, 2, - VertexBuffer::AttributeType::FLOAT4) + VertexBuffer::AttributeType::FLOAT) .attribute(VertexAttribute::CUSTOM0, 3, + VertexBuffer::AttributeType::FLOAT4) + .attribute(VertexAttribute::CUSTOM7, 4, VertexBuffer::AttributeType::FLOAT4); if (sh_degree >= 1) { - buffer_builder.attribute(VertexAttribute::CUSTOM1, 4, + buffer_builder.attribute(VertexAttribute::CUSTOM1, 5, VertexBuffer::AttributeType::FLOAT4); - buffer_builder.attribute(VertexAttribute::CUSTOM2, 5, + buffer_builder.attribute(VertexAttribute::CUSTOM2, 6, VertexBuffer::AttributeType::FLOAT4); - buffer_builder.attribute(VertexAttribute::CUSTOM3, 6, + buffer_builder.attribute(VertexAttribute::CUSTOM3, 7, VertexBuffer::AttributeType::FLOAT4); } if (sh_degree == 2) { - buffer_builder.attribute(VertexAttribute::CUSTOM4, 7, + buffer_builder.attribute(VertexAttribute::CUSTOM4, 8, VertexBuffer::AttributeType::FLOAT4); - buffer_builder.attribute(VertexAttribute::CUSTOM5, 8, + buffer_builder.attribute(VertexAttribute::CUSTOM5, 9, VertexBuffer::AttributeType::FLOAT4); - buffer_builder.attribute(VertexAttribute::CUSTOM6, 9, + buffer_builder.attribute(VertexAttribute::CUSTOM6, 10, VertexBuffer::AttributeType::FLOAT4); } @@ -141,14 +143,15 @@ TGaussianSplatBuffersBuilder::ConstructBuffers() { GeometryBuffersBuilder::DeallocateBuffer); vbuf->setBufferAt(engine, 1, std::move(scale_descriptor)); - const size_t rot_array_size = n_vertices * 4 * sizeof(float); - float* rot_array = static_cast(malloc(rot_array_size)); - std::memcpy(rot_array, geometry_.GetPointAttr("rot").GetDataPtr(), - rot_array_size); - VertexBuffer::BufferDescriptor rot_descriptor( - rot_array, rot_array_size, + // We need to allocate a buffer for TANGENTS; otherwise, Filament will issue + // a warning. + const size_t empty_array_size = n_vertices * sizeof(float); + float* empty_array = static_cast(malloc(empty_array_size)); + std::memset(empty_array, 0, empty_array_size); + VertexBuffer::BufferDescriptor empty_descriptor( + empty_array, empty_array_size, GeometryBuffersBuilder::DeallocateBuffer); - vbuf->setBufferAt(engine, 2, std::move(rot_descriptor)); + vbuf->setBufferAt(engine, 2, std::move(empty_descriptor)); const size_t color_array_size = n_vertices * 4 * sizeof(float); float* color_array = static_cast(malloc(color_array_size)); @@ -163,11 +166,19 @@ TGaussianSplatBuffersBuilder::ConstructBuffers() { GeometryBuffersBuilder::DeallocateBuffer); vbuf->setBufferAt(engine, 3, std::move(color_descriptor)); + const size_t rot_array_size = n_vertices * 4 * sizeof(float); + float* rot_array = static_cast(malloc(rot_array_size)); + std::memcpy(rot_array, geometry_.GetPointAttr("rot").GetDataPtr(), + rot_array_size); + VertexBuffer::BufferDescriptor rot_descriptor( + rot_array, rot_array_size, + GeometryBuffersBuilder::DeallocateBuffer); + vbuf->setBufferAt(engine, 4, std::move(rot_descriptor)); + int data_count_in_one_buffer = 4; const size_t f_rest_array_size = n_vertices * data_count_in_one_buffer * sizeof(float); - const size_t custom_buffer_start_index = 4; - + const size_t custom_buffer_start_index = 5; float* f_rest_src = (f_rest_buffer_count > 0) ? geometry_.GetPointAttr("f_rest").GetDataPtr() @@ -183,8 +194,7 @@ TGaussianSplatBuffersBuilder::ConstructBuffers() { f_rest_buffer_count * data_count_in_one_buffer); copy_data_size = n_vertices * remaining_count_in_last_iter * sizeof(float); - std::memset(f_rest_src + i * n_vertices * data_count_in_one_buffer, - 0, f_rest_array_size); + std::memset(f_rest_array, 0, f_rest_array_size); } std::memcpy(f_rest_array, From a24fda5dd9550331f074704c7f90d77c50232805 Mon Sep 17 00:00:00 2001 From: Zhang Yang Date: Tue, 18 Mar 2025 14:49:22 +0800 Subject: [PATCH 24/28] upate sh_degree in MapRecord --- cpp/open3d/visualization/rendering/MaterialRecord.h | 2 +- cpp/open3d/visualization/visualizer/O3DVisualizer.cpp | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cpp/open3d/visualization/rendering/MaterialRecord.h b/cpp/open3d/visualization/rendering/MaterialRecord.h index 61970ae5b90..ef1b174d35a 100644 --- a/cpp/open3d/visualization/rendering/MaterialRecord.h +++ b/cpp/open3d/visualization/rendering/MaterialRecord.h @@ -84,7 +84,7 @@ struct MaterialRecord { float ground_plane_axis = 0.f; // 0: XZ; >0: XY; <0: YZ // This is only used in gaussian splat. - int sh_degree = 2; + int sh_degree = 0; // Generic material properties std::unordered_map generic_params; diff --git a/cpp/open3d/visualization/visualizer/O3DVisualizer.cpp b/cpp/open3d/visualization/visualizer/O3DVisualizer.cpp index dae6bf9fbae..f5c125a1681 100644 --- a/cpp/open3d/visualization/visualizer/O3DVisualizer.cpp +++ b/cpp/open3d/visualization/visualizer/O3DVisualizer.cpp @@ -937,6 +937,7 @@ Ctrl-alt-click to polygon select)"; } if (is_gaussian_splat) { mat.shader = kShaderGaussianSplat; + mat.sh_degree = t_cloud->GaussianSplatGetSHOrder(); } mat.point_size = ConvertToScaledPixels(ui_state_.point_size); From 67c7c3e5e7377bad44b869380aa4b033d02cfc8d Mon Sep 17 00:00:00 2001 From: Zhang Yang Date: Tue, 25 Mar 2025 10:30:24 +0800 Subject: [PATCH 25/28] set sh degree as 2 in ConstructBuffers --- cpp/open3d/t/geometry/PointCloud.cpp | 8 -------- .../rendering/filament/GaussianSplatBuffers.cpp | 6 ++++++ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/cpp/open3d/t/geometry/PointCloud.cpp b/cpp/open3d/t/geometry/PointCloud.cpp index 6988376ada3..e56494e891a 100644 --- a/cpp/open3d/t/geometry/PointCloud.cpp +++ b/cpp/open3d/t/geometry/PointCloud.cpp @@ -1406,15 +1406,7 @@ int PointCloud::GaussianSplatGetSHOrder() const { "({}), expected 0, 3, 8 or 15.", Nc); } - deg = deg - 1; - if (deg > 2) { - utility::LogWarning( - "SH degree higher than 2 is not supported. The SH degree " - "of in input data is {}, we set it as 2.", - deg); - deg = 2; - } return deg; } diff --git a/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp b/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp index 181a4184b57..e5dc7b1c92d 100644 --- a/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp +++ b/cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp @@ -72,6 +72,12 @@ TGaussianSplatBuffersBuilder::ConstructBuffers() { const size_t n_vertices = points.GetLength(); int sh_degree = geometry_.GaussianSplatGetSHOrder(); + if (sh_degree > 2) { + utility::LogWarning( + "Rendering for Gaussian splats with SH degrees higher than 2 " + "is not supported. They are processed as SH degree 2."); + sh_degree = 2; + } int f_rest_coeffs_count = sh_degree * (sh_degree + 2) * 3; int f_rest_buffer_count = (f_rest_coeffs_count % 4 == 0) From 78cb4e2a4c588fcafea9ea22d4fa9217a06b8609 Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Thu, 27 Mar 2025 22:22:35 -0700 Subject: [PATCH 26/28] CI check fix WIP --- cpp/tests/Tests.h | 12 ++++++ cpp/tests/t/io/PointCloudIO.cpp | 75 +++++++++++++++++---------------- 2 files changed, 50 insertions(+), 37 deletions(-) diff --git a/cpp/tests/Tests.h b/cpp/tests/Tests.h index f5c0cfd5b97..62197ae0699 100644 --- a/cpp/tests/Tests.h +++ b/cpp/tests/Tests.h @@ -34,5 +34,17 @@ const Eigen::Vector2i Zero2i = Eigen::Vector2i::Zero(); // Mechanism for reporting unit tests for which there is no implementation yet. void NotImplemented(); +#define AllCloseOrMaxError(Arr1, Arr2, rtol, atol) \ + EXPECT_TRUE(Arr1.AllClose(Arr2, rtol, atol)) << fmt::format( \ + "Tensors are not close wrt (relative, absolute) tolerance ({}, " \ + "{}). Max error: {} ", \ + rtol, atol, \ + (Arr1 - Arr2) \ + .Abs() \ + .Flatten() \ + .Max({0}) \ + .To(core::Float32) \ + .Item()); + } // namespace tests } // namespace open3d diff --git a/cpp/tests/t/io/PointCloudIO.cpp b/cpp/tests/t/io/PointCloudIO.cpp index 5ebd2bc8ddf..389406a2310 100644 --- a/cpp/tests/t/io/PointCloudIO.cpp +++ b/cpp/tests/t/io/PointCloudIO.cpp @@ -435,18 +435,18 @@ TEST_P(PointCloudIOPermuteDevices, ReadWrite3DGSPointCloudPLY) { auto num_gaussians_new = pcd_ply_binary.GetPointPositions().GetLength(); EXPECT_EQ(num_gaussians_base, num_gaussians_new); - EXPECT_TRUE(pcd_ply.GetPointPositions().AllClose( - pcd_ply_binary.GetPointPositions())); - EXPECT_TRUE(pcd_ply.GetPointAttr("scale").AllClose( - pcd_ply_binary.GetPointAttr("scale"))); - EXPECT_TRUE(pcd_ply.GetPointAttr("opacity").AllClose( - pcd_ply_binary.GetPointAttr("opacity"))); - EXPECT_TRUE(pcd_ply.GetPointAttr("rot").AllClose( - pcd_ply_binary.GetPointAttr("rot"))); - EXPECT_TRUE(pcd_ply.GetPointAttr("f_dc").AllClose( - pcd_ply_binary.GetPointAttr("f_dc"))); - EXPECT_TRUE(pcd_ply.GetPointAttr("f_rest").AllClose( - pcd_ply_binary.GetPointAttr("f_rest"))); + AllCloseOrMaxError(pcd_ply.GetPointPositions(), + pcd_ply_binary.GetPointPositions(), 1e-5, 1e-8); + AllCloseOrMaxError(pcd_ply.GetPointAttr("scale"), + pcd_ply_binary.GetPointAttr("scale"), 1e-5, 1e-8); + AllCloseOrMaxError(pcd_ply.GetPointAttr("opacity"), + pcd_ply_binary.GetPointAttr("opacity"), 1e-5, 1e-8); + AllCloseOrMaxError(pcd_ply.GetPointAttr("rot"), + pcd_ply_binary.GetPointAttr("rot"), 1e-5, 1e-8); + AllCloseOrMaxError(pcd_ply.GetPointAttr("f_dc"), + pcd_ply_binary.GetPointAttr("f_dc"), 1e-5, 1e-8); + AllCloseOrMaxError(pcd_ply.GetPointAttr("f_rest"), + pcd_ply_binary.GetPointAttr("f_rest"), 1e-5, 1e-8); auto opacity = pcd_ply.GetPointAttr("opacity"); // Error if the shape of the attribute is not 2D with len = num_points @@ -597,19 +597,19 @@ TEST(TPointCloudIO, ReadWrite3DGSPLYToSPLAT) { t::io::ReadPointCloud(filename_splat, pcd_splat, {"auto", false, false, true}); - EXPECT_TRUE(pcd_ply.GetPointPositions().AllClose( - pcd_splat.GetPointPositions())); - EXPECT_TRUE(pcd_ply.GetPointAttr("scale").AllClose( - pcd_splat.GetPointAttr("scale"))); - EXPECT_TRUE(pcd_ply.GetPointAttr("opacity").AllClose( - pcd_splat.GetPointAttr("opacity"), 0, - 0.01)); // expect quantization errors - EXPECT_TRUE(pcd_ply.GetPointAttr("rot").AllClose( - pcd_splat.GetPointAttr("rot"), 0, - 0.01)); // expect quantization errors - EXPECT_TRUE(pcd_ply.GetPointAttr("f_dc").AllClose( - pcd_splat.GetPointAttr("f_dc"), 0, - 0.01)); // expect quantization errors + AllCloseOrMaxError(pcd_ply.GetPointPositions(), + pcd_splat.GetPointPositions(), 1e-5, 1e-8); + AllCloseOrMaxError(pcd_ply.GetPointAttr("scale"), + pcd_splat.GetPointAttr("scale"), 1e-5, 1e-8); + AllCloseOrMaxError(pcd_ply.GetPointAttr("opacity"), + pcd_splat.GetPointAttr("opacity"), 0, + 0.01); // expect quantization errors + AllCloseOrMaxError(pcd_ply.GetPointAttr("rot"), + pcd_splat.GetPointAttr("rot"), 0, + 0.01); // expect quantization errors + AllCloseOrMaxError(pcd_ply.GetPointAttr("f_dc"), + pcd_splat.GetPointAttr("f_dc"), 0, + 0.01); // expect quantization errors EXPECT_FALSE(pcd_splat.HasPointAttr("f_rest")); } @@ -658,18 +658,19 @@ TEST(TPointCloudIO, ReadWrite3DGSSPLAT) { auto num_gaussians_new = pcd_new.GetPointPositions().GetLength(); EXPECT_EQ(num_gaussians_base, num_gaussians_new); - EXPECT_TRUE( - pcd_base.GetPointPositions().AllClose(pcd_new.GetPointPositions())); - EXPECT_TRUE(pcd_base.GetPointAttr("scale").AllClose( - pcd_new.GetPointAttr("scale"))); - EXPECT_TRUE(pcd_base.GetPointAttr("opacity").AllClose( - pcd_new.GetPointAttr("opacity"))); - EXPECT_TRUE(pcd_base.GetPointAttr("rot").AllClose( - pcd_new.GetPointAttr("rot"), 0, - 0.004)); // expect quantization errors - EXPECT_TRUE(pcd_base.GetPointAttr("f_dc").AllClose( - pcd_new.GetPointAttr("f_dc"), 0, - 0.004)); // expect quantization errors + AllCloseOrMaxError(pcd_base.GetPointPositions(), + pcd_new.GetPointPositions(), 1e-5, 1e-8); + AllCloseOrMaxError(pcd_base.GetPointAttr("scale"), + pcd_new.GetPointAttr("scale"), 1e-5, 1e-8); + AllCloseOrMaxError(pcd_base.GetPointAttr("opacity"), + pcd_new.GetPointAttr("opacity"), 0, + 0.01); // expect quantization errors + AllCloseOrMaxError(pcd_base.GetPointAttr("rot"), + pcd_new.GetPointAttr("rot"), 0, + 0.01); // expect quantization errors + AllCloseOrMaxError(pcd_base.GetPointAttr("f_dc"), + pcd_new.GetPointAttr("f_dc"), 0, + 0.01); // expect quantization errors } } // namespace tests From 1fabac67fcea063679fb8e6367578152302f1bba Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Fri, 28 Mar 2025 23:31:52 -0700 Subject: [PATCH 27/28] macOS, Win CI check fix 2 --- cpp/open3d/t/io/file_format/FileSPLAT.cpp | 28 ++-- cpp/tests/Tests.h | 7 +- cpp/tests/t/io/PointCloudIO.cpp | 169 +++++++++++----------- 3 files changed, 107 insertions(+), 97 deletions(-) diff --git a/cpp/open3d/t/io/file_format/FileSPLAT.cpp b/cpp/open3d/t/io/file_format/FileSPLAT.cpp index 89d3301b12a..5084212399f 100644 --- a/cpp/open3d/t/io/file_format/FileSPLAT.cpp +++ b/cpp/open3d/t/io/file_format/FileSPLAT.cpp @@ -76,7 +76,6 @@ std::vector SortedSplatIndices(geometry::TensorMap &t_map) { return score_left < score_right; // Sort in descending order }); - return indices; } @@ -179,7 +178,7 @@ bool ReadPointCloudFromSPLAT(const std::string &filename, quat_norm += rot_float[i] * rot_float[i]; } quat_norm = sqrt(quat_norm); - if (quat_norm > 1e-6) { + if (quat_norm > std::numeric_limits::epsilon()) { for (int i = 0; i < 4; i++) { rot_float[i] /= quat_norm; } @@ -199,7 +198,7 @@ bool ReadPointCloudFromSPLAT(const std::string &filename, reporter.Finish(); return true; } catch (const std::exception &e) { - utility::LogError("Read SPLAT failed: {}", e.what()); + utility::LogError("Read SPLAT file {} failed: {}", filename, e.what()); } return false; } @@ -251,8 +250,8 @@ bool WritePointCloudToSPLAT(const std::string &filename, splat_file.exceptions(std::ofstream::badbit); // failbit not set for // binary IO errors } catch (const std::ios_base::failure &) { - utility::LogError("Write SPLAT failed: unable to open file: {}.", - filename); + utility::LogWarning("Write SPLAT failed: unable to open file: {}.", + filename); return false; } @@ -287,13 +286,19 @@ bool WritePointCloudToSPLAT(const std::string &filename, Eigen::Vector4f rot{rot_ptr[rot_offset], rot_ptr[rot_offset + 1], rot_ptr[rot_offset + 2], rot_ptr[rot_offset + 3]}; + if (auto quat_norm = rot.norm(); + quat_norm > std::numeric_limits::epsilon()) { + rot /= quat_norm; + } else { + rot = {1.f, 0.f, 0.f, 0.f}; // wxyz quaternion + } // offset should be 127, but we follow the reference // antimatter/convert.py code - rot = (rot.normalized() * 128.0).array().round() + 128.0; - auto int8_rot = - rot.cwiseMin(255.0).cwiseMax(0.0).cast().eval(); - splat_file.write(reinterpret_cast(int8_rot.data()), - 4 * sizeof(int8_t)); + rot = (rot * 128.0).array().round() + 128.0; + auto uint8_rot = + rot.cwiseMin(255.0).cwiseMax(0.0).cast().eval(); + splat_file.write(reinterpret_cast(uint8_rot.data()), + 4 * sizeof(uint8_t)); if (i % 1000 == 0) { reporter.Update(i); @@ -303,7 +308,8 @@ bool WritePointCloudToSPLAT(const std::string &filename, reporter.Finish(); return true; } catch (const std::ios_base::failure &e) { - utility::LogError("Write SPLAT failed: {}", e.what()); + utility::LogWarning("Write SPLAT to file {} failed: {}", filename, + e.what()); return false; } } diff --git a/cpp/tests/Tests.h b/cpp/tests/Tests.h index 62197ae0699..ba201adfea6 100644 --- a/cpp/tests/Tests.h +++ b/cpp/tests/Tests.h @@ -34,17 +34,18 @@ const Eigen::Vector2i Zero2i = Eigen::Vector2i::Zero(); // Mechanism for reporting unit tests for which there is no implementation yet. void NotImplemented(); -#define AllCloseOrMaxError(Arr1, Arr2, rtol, atol) \ +#define AllCloseOrShow(Arr1, Arr2, rtol, atol) \ EXPECT_TRUE(Arr1.AllClose(Arr2, rtol, atol)) << fmt::format( \ "Tensors are not close wrt (relative, absolute) tolerance ({}, " \ - "{}). Max error: {} ", \ + "{}). Max error: {}\n{}\n{}", \ rtol, atol, \ (Arr1 - Arr2) \ .Abs() \ .Flatten() \ .Max({0}) \ .To(core::Float32) \ - .Item()); + .Item(), \ + Arr1.ToString(), Arr2.ToString()); } // namespace tests } // namespace open3d diff --git a/cpp/tests/t/io/PointCloudIO.cpp b/cpp/tests/t/io/PointCloudIO.cpp index 389406a2310..54a080cac1c 100644 --- a/cpp/tests/t/io/PointCloudIO.cpp +++ b/cpp/tests/t/io/PointCloudIO.cpp @@ -217,40 +217,45 @@ TEST(TPointCloudIO, ReadPointCloudFromPLY3) { namespace { // Test ply file containig 3DGS data -const char test_3dgs_ply_data[] = R"(ply -format ascii 1.0 -element vertex 2 -property float x -property float y -property float z -property float nx -property float ny -property float nz -property float f_dc_0 -property float f_dc_1 -property float f_dc_2 -property float f_rest_0 -property float f_rest_1 -property float f_rest_2 -property float f_rest_3 -property float f_rest_4 -property float f_rest_5 -property float f_rest_6 -property float f_rest_7 -property float f_rest_8 -property float opacity -property float scale_0 -property float scale_1 -property float scale_2 -property float rot_0 -property float rot_1 -property float rot_2 -property float rot_3 -end_header -0.7236 -0.52572 -0.44721 0.48902 -0.48306 -0.7263 -1.20846 0.45058 -0.98568 -0.15648 0.03506 -0.07857 0.03506 -0.06857 0.06506 -0.03857 0.14588 -0.25489 0.56895 2.56841 0.58956 1.54784 -0.34619 -0.7938 -0.48108 0.1364 -0.6598 -1.42875 -2.85722 -0.69152 0.69793 -0.18625 0.24585 -0.3305 -1.58646 -0.15865 -0.05305 -0.12865 -0.08305 -0.11865 -0.09305 0.05648 -0.28579 -0.04457 0.33395 1.58847 2.55896 0.58984 0.55591 0.27858 -0.50835 -0.59577 -)"; - +const char test_3dgs_ply_data[] = + "ply \n" + "format ascii 1.0 \n" + "element vertex 2 \n" + "property float x \n" + "property float y \n" + "property float z \n" + "property float nx \n" + "property float ny \n" + "property float nz \n" + "property float f_dc_0 \n" + "property float f_dc_1 \n" + "property float f_dc_2 \n" + "property float f_rest_0 \n" + "property float f_rest_1 \n" + "property float f_rest_2 \n" + "property float f_rest_3 \n" + "property float f_rest_4 \n" + "property float f_rest_5 \n" + "property float f_rest_6 \n" + "property float f_rest_7 \n" + "property float f_rest_8 \n" + "property float opacity \n" + "property float scale_0 \n" + "property float scale_1 \n" + "property float scale_2 \n" + "property float rot_0 \n" + "property float rot_1 \n" + "property float rot_2 \n" + "property float rot_3 \n" + "end_header \n" + "0.7236 -0.52572 -0.44721 0.48902 -0.48306 -0.7263 -1.20846 " + "0.45058 -0.98568 -0.15648 0.03506 -0.07857 0.03506 -0.06857 " + "0.06506 -0.03857 0.14588 -0.25489 0.56895 2.56841 0.58956 1.54784 " + "-0.34619 -0.7938 -0.48108 0.1364 \n" + "0.6598 -1.42875 -2.85722 -0.69152 0.69793 -0.18625 0.24585 -0.3305 " + "-1.58646 -0.15865 -0.05305 -0.12865 -0.08305 -0.11865 -0.09305 " + "0.05648 -0.28579 -0.04457 0.33395 1.58847 2.55896 0.58984 " + "0.55591 0.27858 -0.50835 -0.59577 \n"; } // namespace // Reading ascii and check for 3DGS attributes. @@ -435,18 +440,18 @@ TEST_P(PointCloudIOPermuteDevices, ReadWrite3DGSPointCloudPLY) { auto num_gaussians_new = pcd_ply_binary.GetPointPositions().GetLength(); EXPECT_EQ(num_gaussians_base, num_gaussians_new); - AllCloseOrMaxError(pcd_ply.GetPointPositions(), - pcd_ply_binary.GetPointPositions(), 1e-5, 1e-8); - AllCloseOrMaxError(pcd_ply.GetPointAttr("scale"), - pcd_ply_binary.GetPointAttr("scale"), 1e-5, 1e-8); - AllCloseOrMaxError(pcd_ply.GetPointAttr("opacity"), - pcd_ply_binary.GetPointAttr("opacity"), 1e-5, 1e-8); - AllCloseOrMaxError(pcd_ply.GetPointAttr("rot"), - pcd_ply_binary.GetPointAttr("rot"), 1e-5, 1e-8); - AllCloseOrMaxError(pcd_ply.GetPointAttr("f_dc"), - pcd_ply_binary.GetPointAttr("f_dc"), 1e-5, 1e-8); - AllCloseOrMaxError(pcd_ply.GetPointAttr("f_rest"), - pcd_ply_binary.GetPointAttr("f_rest"), 1e-5, 1e-8); + AllCloseOrShow(pcd_ply.GetPointPositions(), + pcd_ply_binary.GetPointPositions(), 1e-5, 1e-8); + AllCloseOrShow(pcd_ply.GetPointAttr("scale"), + pcd_ply_binary.GetPointAttr("scale"), 1e-5, 1e-8); + AllCloseOrShow(pcd_ply.GetPointAttr("opacity"), + pcd_ply_binary.GetPointAttr("opacity"), 1e-5, 1e-8); + AllCloseOrShow(pcd_ply.GetPointAttr("rot"), + pcd_ply_binary.GetPointAttr("rot"), 1e-5, 1e-8); + AllCloseOrShow(pcd_ply.GetPointAttr("f_dc"), + pcd_ply_binary.GetPointAttr("f_dc"), 1e-5, 1e-8); + AllCloseOrShow(pcd_ply.GetPointAttr("f_rest"), + pcd_ply_binary.GetPointAttr("f_rest"), 1e-5, 1e-8); auto opacity = pcd_ply.GetPointAttr("opacity"); // Error if the shape of the attribute is not 2D with len = num_points @@ -597,19 +602,19 @@ TEST(TPointCloudIO, ReadWrite3DGSPLYToSPLAT) { t::io::ReadPointCloud(filename_splat, pcd_splat, {"auto", false, false, true}); - AllCloseOrMaxError(pcd_ply.GetPointPositions(), - pcd_splat.GetPointPositions(), 1e-5, 1e-8); - AllCloseOrMaxError(pcd_ply.GetPointAttr("scale"), - pcd_splat.GetPointAttr("scale"), 1e-5, 1e-8); - AllCloseOrMaxError(pcd_ply.GetPointAttr("opacity"), - pcd_splat.GetPointAttr("opacity"), 0, - 0.01); // expect quantization errors - AllCloseOrMaxError(pcd_ply.GetPointAttr("rot"), - pcd_splat.GetPointAttr("rot"), 0, - 0.01); // expect quantization errors - AllCloseOrMaxError(pcd_ply.GetPointAttr("f_dc"), - pcd_splat.GetPointAttr("f_dc"), 0, - 0.01); // expect quantization errors + AllCloseOrShow(pcd_ply.GetPointPositions(), pcd_splat.GetPointPositions(), + 1e-5, 1e-8); + AllCloseOrShow(pcd_ply.GetPointAttr("scale"), + pcd_splat.GetPointAttr("scale"), 1e-5, 1e-8); + AllCloseOrShow(pcd_ply.GetPointAttr("opacity"), + pcd_splat.GetPointAttr("opacity"), 0, + 0.01); // expect quantization errors + AllCloseOrShow(pcd_ply.GetPointAttr("rot"), pcd_splat.GetPointAttr("rot"), + 0, + 0.01); // expect quantization errors + AllCloseOrShow(pcd_ply.GetPointAttr("f_dc"), pcd_splat.GetPointAttr("f_dc"), + 0, + 0.01); // expect quantization errors EXPECT_FALSE(pcd_splat.HasPointAttr("f_rest")); } @@ -623,20 +628,19 @@ TEST(TPointCloudIO, ReadWrite3DGSSPLAT) { std::string new_filename = utility::filesystem::GetTempDirectoryPath() + "/new_test_read.splat"; - // Write a small splat file. - // This is the same point cloud as test_3dgs_ply_data (without f_rest), - // converted to splat using the reference code from + // Write a small splat file. This is the same point cloud as + // test_3dgs_ply_data (without f_rest), converted to splat using the + // reference code from // github.com/antimatter15/splat/blob/367a9439609d043f1b23a9b455a77a977f2e7758/convert.py - // (March 2025). Converted to C array using xxd. + // (March 2025). Converted to C array with: // xxd -i test_3dgs_data.splat > test_3dgs_data_splat.h + // clang-format off const unsigned char output_splat[64] = { - 0xd9, 0x3d, 0x39, 0x3f, 0x96, 0x95, 0x06, 0xbf, 0xb6, 0xf8, 0xe4, - 0xbe, 0x96, 0xb8, 0x50, 0x41, 0x16, 0xcf, 0xe6, 0x3f, 0x16, 0x71, - 0x96, 0x40, 0x29, 0xa0, 0x39, 0xa3, 0x54, 0x1a, 0x42, 0x91, 0xa7, - 0xe8, 0x28, 0x3f, 0x48, 0xe1, 0xb6, 0xbf, 0xb1, 0xdc, 0x36, 0xc0, - 0x18, 0xae, 0x9c, 0x40, 0x08, 0xc2, 0x4e, 0x41, 0xa2, 0xdf, 0xe6, - 0x3f, 0x91, 0x68, 0x0d, 0x95, 0xc7, 0xa4, 0x3f, 0x34}; - + 0xd9, 0x3d, 0x39, 0x3f, 0x96, 0x95, 0x06, 0xbf, 0xb6, 0xf8, 0xe4, 0xbe, 0x96, 0xb8, 0x50, 0x41, + 0x16, 0xcf, 0xe6, 0x3f, 0x16, 0x71, 0x96, 0x40, 0x29, 0xa0, 0x39, 0xa3, 0x54, 0x1a, 0x42, 0x91, + 0xa7, 0xe8, 0x28, 0x3f, 0x48, 0xe1, 0xb6, 0xbf, 0xb1, 0xdc, 0x36, 0xc0, 0x18, 0xae, 0x9c, 0x40, + 0x08, 0xc2, 0x4e, 0x41, 0xa2, 0xdf, 0xe6, 0x3f, 0x91, 0x68, 0x0d, 0x95, 0xc7, 0xa4, 0x3f, 0x34}; + // clang-format on { std::ofstream outfile(filename, std::ios::binary); // Open in binary mode @@ -658,19 +662,18 @@ TEST(TPointCloudIO, ReadWrite3DGSSPLAT) { auto num_gaussians_new = pcd_new.GetPointPositions().GetLength(); EXPECT_EQ(num_gaussians_base, num_gaussians_new); - AllCloseOrMaxError(pcd_base.GetPointPositions(), - pcd_new.GetPointPositions(), 1e-5, 1e-8); - AllCloseOrMaxError(pcd_base.GetPointAttr("scale"), - pcd_new.GetPointAttr("scale"), 1e-5, 1e-8); - AllCloseOrMaxError(pcd_base.GetPointAttr("opacity"), - pcd_new.GetPointAttr("opacity"), 0, - 0.01); // expect quantization errors - AllCloseOrMaxError(pcd_base.GetPointAttr("rot"), - pcd_new.GetPointAttr("rot"), 0, - 0.01); // expect quantization errors - AllCloseOrMaxError(pcd_base.GetPointAttr("f_dc"), - pcd_new.GetPointAttr("f_dc"), 0, - 0.01); // expect quantization errors + AllCloseOrShow(pcd_base.GetPointPositions(), pcd_new.GetPointPositions(), + 1e-5, 1e-8); + AllCloseOrShow(pcd_base.GetPointAttr("scale"), + pcd_new.GetPointAttr("scale"), 1e-5, 1e-8); + AllCloseOrShow(pcd_base.GetPointAttr("opacity"), + pcd_new.GetPointAttr("opacity"), 0, + 0.01); // expect quantization errors + AllCloseOrShow(pcd_base.GetPointAttr("rot"), pcd_new.GetPointAttr("rot"), 0, + 0.01); // expect quantization errors + AllCloseOrShow(pcd_base.GetPointAttr("f_dc"), pcd_new.GetPointAttr("f_dc"), + 0, + 0.01); // expect quantization errors } } // namespace tests From abe6ac42acf1fc7ab502fb7bbe1f062bb22152ee Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Sat, 29 Mar 2025 08:49:00 -0700 Subject: [PATCH 28/28] docs build fix. librealsense needs old cmake. --- util/ci_utils.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/util/ci_utils.sh b/util/ci_utils.sh index 0d798ca9139..ba089901009 100644 --- a/util/ci_utils.sh +++ b/util/ci_utils.sh @@ -76,7 +76,7 @@ install_python_dependencies() { fi if [ "$BUILD_PYTORCH_OPS" == "ON" ]; then # ML/requirements-torch.txt if [[ "$OSTYPE" == "linux-gnu"* && "$BUILD_SYCL_MODULE" == "OFF" ]]; then - python -m pip install -U "${TORCH_GLNX}" -f "$TORCH_REPO_URL" + python -m pip install -U "${TORCH_GLNX}" -f "$TORCH_REPO_URL" python -m pip install tensorboard elif [[ "$OSTYPE" == "linux-gnu"* && "$BUILD_SYCL_MODULE" == "ON" ]]; then python -m pip install -U "${TORCH_GLNX}.cxx11.abi" -i "$TORCH_CXX11_URL" @@ -356,7 +356,9 @@ install_docs_dependencies() { command -v python python -V python -m pip install -U -q "pip==$PIP_VER" - which cmake || python -m pip install -U -q cmake + # cmake 4.0 breaks librealsense. Remove restriction when librealsense is + # updated. + which cmake || python -m pip install -U -q cmake <4.0 python -m pip install -U -q -r "${OPEN3D_SOURCE_ROOT}/python/requirements_build.txt" if [[ -d "$1" ]]; then OPEN3D_ML_ROOT="$1"