[Do not merge] enable gaussian rendering based on filament

cpp/open3d/t/geometry/PointCloud.cpp

@@ -1369,6 +1369,16 @@ core::Tensor PointCloud::ComputeMetrics(const PointCloud &pcd2,
                                 metrics, params);
 }
 
+bool PointCloud::IsGaussianSplat() const {
+    std::vector<std::string> keys_to_check = {"f_dc", "opacity", "rot", "scale"};
+    for (const auto& key : keys_to_check) {
+        if (point_attr_.find(key) == point_attr_.end()) {
+            return false;
+        }
+    }
+    return true;
+}
+
 }  // namespace geometry
 }  // namespace t
 }  // namespace open3d

cpp/open3d/t/geometry/PointCloud.h

@@ -738,6 +738,7 @@ class PointCloud : public Geometry, public DrawableGeometry {
             std::vector<Metric> metrics = {Metric::ChamferDistance},
             MetricParameters params = MetricParameters()) const;
 
+    bool IsGaussianSplat() const;
 protected:
     core::Device device_ = core::Device("CPU:0");
     TensorMap point_attr_;

cpp/open3d/visualization/CMakeLists.txt

@@ -80,6 +80,7 @@ if (BUILD_GUI)
         rendering/filament/LineSetBuffers.cpp
         rendering/filament/PointCloudBuffers.cpp
         rendering/filament/TriangleMeshBuffers.cpp
+        rendering/filament/GaussianSplatBuffers.cpp
     )
 
     target_sources(visualization PRIVATE

cpp/open3d/visualization/rendering/MaterialRecord.h

@@ -83,6 +83,9 @@ struct MaterialRecord {
     // Infinite ground plane
     float ground_plane_axis = 0.f;  // 0: XZ; >0: XY; <0: YZ
 
+    // This is only used in gaussian splat.
+    int sh_degree = 2;
+
     // Generic material properties
     std::unordered_map<std::string, Eigen::Vector4f> generic_params;
     std::unordered_map<std::string, geometry::Image> generic_imgs;

cpp/open3d/visualization/rendering/filament/FilamentGeometryBuffersBuilder.cpp

@@ -250,9 +250,14 @@ std::unique_ptr<GeometryBuffersBuilder> GeometryBuffersBuilder::GetBuilder(
     using GT = t::geometry::Geometry::GeometryType;
 
     switch (geometry.GetGeometryType()) {
-        case GT::PointCloud:
-            return std::make_unique<TPointCloudBuffersBuilder>(
-                    static_cast<const t::geometry::PointCloud&>(geometry));
+        case GT::PointCloud: {
+            const t::geometry::PointCloud& pointcloud = static_cast<const t::geometry::PointCloud&>(geometry);
+            if (pointcloud.IsGaussianSplat()) {
+                return std::make_unique<TGaussianSplatBuffersBuilder>(pointcloud);
+            } else {
+                return std::make_unique<TPointCloudBuffersBuilder>(pointcloud);
+            }
+        }
         case GT::TriangleMesh:
             return std::make_unique<TMeshBuffersBuilder>(
                     static_cast<const t::geometry::TriangleMesh&>(geometry));

cpp/open3d/visualization/rendering/filament/FilamentGeometryBuffersBuilder.h

@@ -172,10 +172,17 @@ class TPointCloudBuffersBuilder : public GeometryBuffersBuilder {
     Buffers ConstructBuffers() override;
     filament::Box ComputeAABB() override;
 
-private:
+protected:
     t::geometry::PointCloud geometry_;
 };
 
+class TGaussianSplatBuffersBuilder : public TPointCloudBuffersBuilder {
+public:
+    explicit TGaussianSplatBuffersBuilder(const t::geometry::PointCloud& geometry);
+
+    Buffers ConstructBuffers() override;
+};
+
 class TLineSetBuffersBuilder : public GeometryBuffersBuilder {
 public:
     explicit TLineSetBuffersBuilder(const t::geometry::LineSet& geometry);

cpp/open3d/visualization/rendering/filament/FilamentResourceManager.cpp

@@ -318,6 +318,8 @@ TextureSettings GetSettingsFromImage(const t::geometry::Image& image,
 
 const MaterialHandle FilamentResourceManager::kDefaultLit =
         MaterialHandle::Next();
+const MaterialHandle FilamentResourceManager::kGaussianSplatShader =
+        MaterialHandle::Next();
 const MaterialHandle FilamentResourceManager::kDefaultLitWithTransparency =
         MaterialHandle::Next();
 const MaterialHandle FilamentResourceManager::kDefaultLitSSR =
@@ -1017,6 +1019,25 @@ void FilamentResourceManager::LoadDefaults() {
     lit_mat->setDefaultParameter("anisotropyMap", texture, default_sampler);
     materials_[kDefaultLit] = BoxResource(lit_mat, engine_);
 
+    const auto gaussian_path = resource_root + "/gaussianSplat.filamat";
+    auto gaussian_mat = LoadMaterialFromFile(gaussian_path, engine_);
+    gaussian_mat->setDefaultParameter("baseColor", filament::RgbType::sRGB,
+                                    default_color);
+    gaussian_mat->setDefaultParameter("baseRoughness", 0.7f);
+    gaussian_mat->setDefaultParameter("reflectance", 0.5f);
+    gaussian_mat->setDefaultParameter("baseMetallic", 0.f);
+    gaussian_mat->setDefaultParameter("clearCoat", 0.f);
+    gaussian_mat->setDefaultParameter("clearCoatRoughness", 0.f);
+    gaussian_mat->setDefaultParameter("anisotropy", 0.f);
+    gaussian_mat->setDefaultParameter("pointSize", 3.f);
+    gaussian_mat->setDefaultParameter("albedo", texture, default_sampler);
+    gaussian_mat->setDefaultParameter("ao_rough_metalMap", texture, default_sampler);
+    gaussian_mat->setDefaultParameter("normalMap", normal_map, default_sampler);
+    gaussian_mat->setDefaultParameter("reflectanceMap", texture, default_sampler);
+
+    gaussian_mat->setDefaultParameter("anisotropyMap", texture, default_sampler);
+    materials_[kGaussianSplatShader] = BoxResource(gaussian_mat, engine_);
+
     const auto lit_trans_path =
             resource_root + "/defaultLitTransparency.filamat";
     auto lit_trans_mat = LoadMaterialFromFile(lit_trans_path, engine_);

cpp/open3d/visualization/rendering/filament/FilamentResourceManager.h

@@ -50,6 +50,7 @@ namespace rendering {
 class FilamentResourceManager {
 public:
     static const MaterialHandle kDefaultLit;
+    static const MaterialHandle kGaussianSplatShader;
     static const MaterialHandle kDefaultLitWithTransparency;
     static const MaterialHandle kDefaultLitSSR;
     static const MaterialHandle kDefaultUnlit;

cpp/open3d/visualization/rendering/filament/FilamentScene.cpp

@@ -102,7 +102,8 @@ std::unordered_map<std::string, MaterialHandle> shader_mappings = {
          ResourceManager::kDefaultUnlitPolygonOffsetShader},
         {"unlitBackground", ResourceManager::kDefaultUnlitBackgroundShader},
         {"infiniteGroundPlane", ResourceManager::kInfinitePlaneShader},
-        {"unlitLine", ResourceManager::kDefaultLineShader}};
+        {"unlitLine", ResourceManager::kDefaultLineShader},
+        {"gaussianSplat", ResourceManager::kGaussianSplatShader}};
 
 MaterialHandle kColorOnlyMesh = ResourceManager::kDefaultUnlit;
 MaterialHandle kPlainMesh = ResourceManager::kDefaultLit;
@@ -846,6 +847,32 @@ void FilamentScene::UpdateDefaultLit(GeometryMaterialInstance& geom_mi) {
             .Finish();
 }
 
+void FilamentScene::UpdateGaussianSplat(GeometryMaterialInstance& geom_mi) {
+    auto& material = geom_mi.properties;
+    auto& maps = geom_mi.maps;
+
+    renderer_.ModifyMaterial(geom_mi.mat_instance)
+            .SetColor("baseColor", material.base_color, false)
+            .SetParameter("pointSize", material.point_size)
+            .SetParameter("baseRoughness", material.base_roughness)
+            .SetParameter("baseMetallic", material.base_metallic)
+            .SetParameter("reflectance", material.base_reflectance)
+            .SetParameter("clearCoat", material.base_clearcoat)
+            .SetParameter("clearCoatRoughness",
+                          material.base_clearcoat_roughness)
+            .SetParameter("anisotropy", material.base_anisotropy)
+            .SetParameter("shDegree", material.sh_degree)
+            .SetTexture("albedo", maps.albedo_map,
+                        rendering::TextureSamplerParameters::Pretty())
+            .SetTexture("normalMap", maps.normal_map,
+                        rendering::TextureSamplerParameters::Pretty())
+            .SetTexture("ao_rough_metalMap", maps.ao_rough_metal_map,
+                        rendering::TextureSamplerParameters::Pretty())
+            .SetTexture("reflectanceMap", maps.reflectance_map,
+                        rendering::TextureSamplerParameters::Pretty())
+            .Finish();
+}
+
 void FilamentScene::UpdateDefaultLitSSR(GeometryMaterialInstance& geom_mi) {
     auto& material = geom_mi.properties;
     auto& maps = geom_mi.maps;
@@ -1131,6 +1158,8 @@ void FilamentScene::UpdateMaterialProperties(RenderableGeometry& geom) {
         UpdateLineShader(geom.mat);
     } else if (props.shader == "unlitPolygonOffset") {
         UpdateUnlitPolygonOffsetShader(geom.mat);
+    } else if (props.shader == "gaussianSplat") {
+        UpdateGaussianSplat(geom.mat);
     } else {
         utility::LogWarning("'{}' is not a valid shader", props.shader);
     }

cpp/open3d/visualization/rendering/filament/FilamentScene.h

@@ -305,6 +305,7 @@ class FilamentScene : public Scene {
                                   bool shader_only = false);
     void UpdateMaterialProperties(RenderableGeometry& geom);
     void UpdateDefaultLit(GeometryMaterialInstance& geom_mi);
+    void UpdateGaussianSplat(GeometryMaterialInstance& geom_mi);
     void UpdateDefaultLitSSR(GeometryMaterialInstance& geom_mi);
     void UpdateDefaultUnlit(GeometryMaterialInstance& geom_mi);
     void UpdateNormalShader(GeometryMaterialInstance& geom_mi);

cpp/open3d/visualization/rendering/filament/GaussianSplatBuffers.cpp

@@ -0,0 +1,187 @@
+// ----------------------------------------------------------------------------
+// -                        Open3D: www.open3d.org                            -
+// ----------------------------------------------------------------------------
+// Copyright (c) 2018-2024 www.open3d.org
+// SPDX-License-Identifier: MIT
+// ----------------------------------------------------------------------------
+
+// 4068: Filament has some clang-specific vectorizing pragma's that MSVC flags
+// 4146: Filament's utils/algorithm.h utils::details::ctz() tries to negate
+//       an unsigned int.
+// 4293: Filament's utils/algorithm.h utils::details::clz() does strange
+//       things with MSVC. Somehow sizeof(unsigned int) > 4, but its size is
+//       32 so that x >> 32 gives a warning. (Or maybe the compiler can't
+//       determine the if statement does not run.)
+#ifdef _MSC_VER
+#pragma warning(push)
+#pragma warning(disable : 4068 4146 4293)
+#endif  // _MSC_VER
+
+#include <filament/IndexBuffer.h>
+#include <filament/VertexBuffer.h>
+#include <geometry/SurfaceOrientation.h>
+
+#ifdef _MSC_VER
+#pragma warning(pop)
+#endif  // _MSC_VER
+
+#include "open3d/geometry/BoundingVolume.h"
+#include "open3d/geometry/PointCloud.h"
+#include "open3d/t/geometry/PointCloud.h"
+#include "open3d/visualization/rendering/filament/FilamentEngine.h"
+#include "open3d/visualization/rendering/filament/FilamentGeometryBuffersBuilder.h"
+#include "open3d/visualization/rendering/filament/FilamentResourceManager.h"
+
+using namespace filament;
+
+namespace open3d {
+namespace visualization {
+namespace rendering {
+TGaussianSplatBuffersBuilder::TGaussianSplatBuffersBuilder(
+        const t::geometry::PointCloud& geometry)
+    : TPointCloudBuffersBuilder(geometry) {
+    if (!geometry.IsGaussianSplat()) {
+        utility::LogWarning(
+            "TGaussianSplatBuffers is constructed for a geometry that is not GaussianSplat."
+        );
+    }
+
+    std::vector<std::string> check_list = {"f_dc", "opacity", "rot", "scale", "f_rest"};
+    for (const auto& check_item : check_list) {
+        if (geometry_.HasPointAttr(check_item) &&
+            geometry_.GetPointAttr(check_item).GetDtype() != core::Float32) {
+            auto check_item_instance = geometry_.GetPointAttr(check_item);
+            utility::LogWarning(
+                    "Tensor gaussian splat {} must have DType of Float32 not {}. "
+                    "Converting.",
+                    check_item, check_item_instance.GetDtype().ToString());
+            geometry_.GetPointAttr(check_item) = check_item_instance.To(core::Float32);
+        }
+    }
+}
+
+GeometryBuffersBuilder::Buffers TGaussianSplatBuffersBuilder::ConstructBuffers() {
+    auto& engine = EngineInstance::GetInstance();
+    auto& resource_mgr = EngineInstance::GetResourceManager();
+
+    const auto& points = geometry_.GetPointPositions();
+    const size_t n_vertices = points.GetLength();
+
+    // we use POSITION for positions, COLOR for scale, TANGENTS for rot 
+    // CUSTOM0 for f_dc and opacity, CUSTOM1-CUSTOM6 for f_rest. 
+    VertexBuffer* vbuf = VertexBuffer::Builder()
+                                 .bufferCount(10)
+                                 .vertexCount(uint32_t(n_vertices))
+                                 .attribute(VertexAttribute::POSITION, 0,
+                                            VertexBuffer::AttributeType::FLOAT3)
+                                 .attribute(VertexAttribute::COLOR, 1,
+                                            VertexBuffer::AttributeType::FLOAT4)
+                                 .attribute(VertexAttribute::TANGENTS, 2,
+                                            VertexBuffer::AttributeType::FLOAT4)
+                                 .attribute(VertexAttribute::CUSTOM0, 3,
+                                            VertexBuffer::AttributeType::FLOAT4)
+                                 .attribute(VertexAttribute::CUSTOM1, 4,
+                                            VertexBuffer::AttributeType::FLOAT4)
+                                 .attribute(VertexAttribute::CUSTOM2, 5,
+                                            VertexBuffer::AttributeType::FLOAT4)
+                                 .attribute(VertexAttribute::CUSTOM3, 6,
+                                            VertexBuffer::AttributeType::FLOAT4)
+                                 .attribute(VertexAttribute::CUSTOM4, 7,
+                                            VertexBuffer::AttributeType::FLOAT4)
+                                 .attribute(VertexAttribute::CUSTOM5, 8,
+                                            VertexBuffer::AttributeType::FLOAT4)
+                                 .attribute(VertexAttribute::CUSTOM6, 9,
+                                            VertexBuffer::AttributeType::FLOAT4)
+                                 .build(engine);
+
+    VertexBufferHandle vb_handle;
+    if (vbuf) {
+        vb_handle = resource_mgr.AddVertexBuffer(vbuf);
+    } else {
+        return {};
+    }
+
+    const size_t vertex_array_size = n_vertices * 3 * sizeof(float);
+    float* vertex_array = static_cast<float*>(malloc(vertex_array_size));
+    memcpy(vertex_array, points.GetDataPtr(), vertex_array_size);
+    VertexBuffer::BufferDescriptor pts_descriptor(
+            vertex_array, vertex_array_size,
+            GeometryBuffersBuilder::DeallocateBuffer);
+    vbuf->setBufferAt(engine, 0, std::move(pts_descriptor));
+
+    const size_t scale_array_size = n_vertices * 4 * sizeof(float);
+    float* scale_array = static_cast<float*>(malloc(scale_array_size));
+    std::memset(scale_array, 0, scale_array_size);
+    if (geometry_.HasPointAttr("scale")) {
+        float* scale_src = static_cast<float*>(geometry_.GetPointAttr("scale").GetDataPtr());
+        for (size_t i = 0; i < n_vertices; i++) {
+            std::memcpy(scale_array + i * 4, scale_src + i * 3, 3*sizeof(float));
+        }
+    }
+    VertexBuffer::BufferDescriptor scale_descriptor(
+            scale_array, scale_array_size, GeometryBuffersBuilder::DeallocateBuffer);
+    vbuf->setBufferAt(engine, 1, std::move(scale_descriptor));
+
+    const size_t rot_array_size = n_vertices * 4 * sizeof(float);
+    float* rot_array = static_cast<float*>(malloc(rot_array_size));
+    if (geometry_.HasPointAttr("rot")) {
+        std::memcpy(rot_array, geometry_.GetPointAttr("rot").GetDataPtr(),
+               rot_array_size);
+    } else {
+        for (size_t i = 0; i < n_vertices * 4; ++i) {
+            rot_array[i] = 1.f;
+        }
+    }
+    VertexBuffer::BufferDescriptor rot_descriptor(
+                rot_array, rot_array_size,
+                GeometryBuffersBuilder::DeallocateBuffer);
+    vbuf->setBufferAt(engine, 2, std::move(rot_descriptor));
+
+    const size_t color_array_size = n_vertices * 4 * sizeof(float);
+    float* color_array = static_cast<float*>(malloc(color_array_size));
+    if (geometry_.HasPointAttr("f_dc") && geometry_.HasPointAttr("opacity")) {
+        float* f_dc_ptr = static_cast<float*>(geometry_.GetPointAttr("f_dc").GetDataPtr());
+        float* opacity_ptr = static_cast<float*>(geometry_.GetPointAttr("opacity").GetDataPtr());
+        for (size_t i = 0; i < n_vertices; i++) {
+            std::memcpy(color_array + i * 4, f_dc_ptr + i * 3, 3 * sizeof(float));
+            std::memcpy(color_array + i * 4 + 3, opacity_ptr + i, sizeof(float));
+        }
+    } else {
+        for (size_t i = 0; i < n_vertices * 4; i++) {
+            color_array[i] = 1.f;
+        }
+    }
+    VertexBuffer::BufferDescriptor color_descriptor(
+                color_array, color_array_size,
+                GeometryBuffersBuilder::DeallocateBuffer);
+    vbuf->setBufferAt(engine, 3, std::move(color_descriptor));
+
+    const size_t f_rest_array_size = n_vertices * 4 * sizeof(float);
+    const size_t custom_buffer_num = 6;
+    const size_t custom_buffer_start_index = 4;
+    for(size_t i = 0; i < custom_buffer_num; i++) {
+        float* f_rest_array = static_cast<float*>(malloc(f_rest_array_size));
+        if (geometry_.HasPointAttr("f_rest")) {
+            float* f_rest_src = static_cast<float*>(geometry_.GetPointAttr("f_rest").GetDataPtr());
+            std::memcpy(f_rest_array, f_rest_src + i * n_vertices * 4, f_rest_array_size);
+        } else {
+            std::memset(f_rest_array, 0, f_rest_array_size);
+        }
+        VertexBuffer::BufferDescriptor f_rest_descriptor(
+                    f_rest_array, f_rest_array_size, GeometryBuffersBuilder::DeallocateBuffer);
+        vbuf->setBufferAt(engine, custom_buffer_start_index + i, std::move(f_rest_descriptor));
+    }
+
+    auto ib_handle = CreateIndexBuffer(n_vertices);
+
+    IndexBufferHandle downsampled_handle;
+    if (n_vertices >= downsample_threshold_) {
+        downsampled_handle =
+                CreateIndexBuffer(n_vertices, downsample_threshold_);
+    }
+
+    return std::make_tuple(vb_handle, ib_handle, downsampled_handle);
+}
+}  // namespace rendering
+}  // namespace visualization
+}  // namespace open3d