add gaussianSplat mat file for shader

cpp/open3d/visualization/gui/CMakeLists.txt

@@ -92,6 +92,7 @@ open3d_add_compiled_materials(materials
     Materials/infiniteGroundPlane.mat
     Materials/normals.mat
     Materials/pointcloud.mat
+    Materials/gaussianSplat.mat
     Materials/ui_blit.mat
     Materials/unlitBackground.mat
     Materials/unlitGradient.mat

cpp/open3d/visualization/gui/Materials/gaussianSplat.mat

@@ -0,0 +1,151 @@
+material {
+    name : defaultLit,
+    shadingModel : lit,
+    doubleSided : true,
+    parameters : [
+        { type : float3,    name : baseColor },
+        { type : float,     name : baseMetallic },
+        { type : float,     name : baseRoughness },
+        { type : float,     name : reflectance },
+        { type : float,     name : pointSize },
+        { type : float,     name : clearCoat },
+        { type : float,     name : clearCoatRoughness },
+        { type : float,     name : anisotropy },
+        { type : sampler2d, name : albedo },
+        { type : sampler2d, name : ao_rough_metalMap },
+        { type : sampler2d, name : normalMap },
+        { type : sampler2d, name : reflectanceMap },
+        { type : sampler2d, name : anisotropyMap },
+        { type : int,       name : shDegree }
+    ],
+    variables : [
+        colorFromSh,
+        test1,
+        test2,
+        test3
+    ],
+    requires : [
+        color,    // scale
+        tangents,
+        custom0,  // f_dc and opacity
+        custom1,  // custom1-custom6 store f_rest_0-f_rest_23 SH coeffs
+        custom2,
+        custom3,
+        custom4,
+        custom5,
+        custom6,
+        custom7   // rot
+    ]
+}
+
+vertex {
+    #include "../../shader/glsl/ShCoeffsToColorFast.glsl"
+
+    void materialVertex(inout MaterialVertexInputs material) {
+        gl_PointSize = materialParams.pointSize;
+
+        // refering to https://google.github.io/filament/Materials.html#materialdefinitions/shaderpublicapis/vertexonly,
+        // the worldPosition coordinate in the vertex shader is shifted by the camera position.
+        vec3 dir = material.worldPosition.xyz;
+        float inorm = inversesqrt(dot(dir, dir));
+        dir = dir * inorm;
+
+        // According to "../../shader/glsl/ShCoeffsToColorFast.glsl", the coeffs should be in the following order:
+        // [Y0_R, Y0_G, Y0_B, Y1m1_R, Y1m1_G, Y1m1_B, Y10_R, Y10_G, Y10_B, Y11_R, Y11_G, Y11_B, Y2m2_R, Y2m2_G, Y2m2_B, ...]
+        vec3 colors;
+
+        float4 f_dc_and_opacity = getCustom0();
+        colors = sh0_coeffs_to_color_fast(f_dc_and_opacity.xyz);
+
+        if (materialParams.shDegree >= 1) {
+            float coeffs_sh1[3 * 3];
+            float4 buffer1 = getCustom1();
+            float4 buffer2 = getCustom2();
+
+            for (int i = 0; i < 4; i++) {
+                coeffs_sh1[i] = buffer1[i];
+                coeffs_sh1[4 + i] = buffer2[i];
+            }
+
+            float4 buffer3 = getCustom3();
+            coeffs_sh1[8] = buffer3[0];
+
+            colors += sh1_coeffs_to_color_fast(dir, coeffs_sh1);
+        }
+
+        if (materialParams.shDegree >= 2) {
+            float coeffs_sh2[5 * 3];
+
+            float4 buffer3 = getCustom3();
+            coeffs_sh2[0] = buffer3[1];
+            coeffs_sh2[1] = buffer3[2];
+            coeffs_sh2[2] = buffer3[3];
+
+            float4 buffer4 = getCustom4();
+            float4 buffer5 = getCustom5();
+            float4 buffer6 = getCustom6();
+
+            for (int i = 0; i < 4; i++) {
+                coeffs_sh2[3 + i] = buffer4[i];
+                coeffs_sh2[3 + 4 + i] = buffer5[i];
+                coeffs_sh2[3 + 2*4 + i] = buffer6[i];
+            }
+
+            colors += sh2_coeffs_to_color_fast(dir, coeffs_sh2);
+        }
+
+        material.colorFromSh.r = colors[0];
+        material.colorFromSh.g = colors[1];
+        material.colorFromSh.b = colors[2];
+        material.colorFromSh.a = 0.0;
+
+        // The following code is just used for verifing we can get data from different buffer.
+        material.test1 = getCustom7();
+        material.test2 = getCustom1();
+        material.test3 = getCustom6();
+    }
+}
+
+fragment {
+    float sRGB_to_linear(float color) {
+        return color <= 0.04045 ? color / 12.92 : pow((color + 0.055) / 1.055, 2.4);
+    }
+
+    float linear_to_sRGB(float color) {
+        return color <= 0.0031308 ? color * 12.92 : 1.055 * pow(color, 1.0 / 2.4) - 0.055;
+    }
+
+    void material(inout MaterialInputs material) {
+        prepareMaterial(material);
+
+        // The following code is only used to test we can get colorFromSh from vertex shader.
+        float r = 0.0;
+        float g = 0.0;
+        float b = 0.0;
+        material.baseColor.rgb = vec3(r, g, b);
+
+        // test case1: verify the effect of 'f_dc'
+        material.baseColor.rgb = variable_colorFromSh.rgb;
+
+        // test case2: Check we can get data 'rot'.
+        // material.baseColor.rgb = variable_test1.rgb;
+
+        // test case3: Check we can get data 'f_rest' of degree 1
+        // material.baseColor.rgb = variable_test2.rgb * 10.0;
+
+        // test case4: Check we can get data 'f_rest' of degree 2
+        // material.baseColor.rgb = variable_test3.rgb * 10.0;
+
+        // test case5: Check we can get data 'scale'
+        // material.baseColor.rgb = getColor().rgb * (-0.05);
+
+        // test case6: test the color space(linear RGB or sRGB)
+        // r = linear_to_sRGB(variable_colorFromSh.r);
+        // g = linear_to_sRGB(variable_colorFromSh.g);
+        // b = linear_to_sRGB(variable_colorFromSh.b);
+        // r = sRGB_to_linear(variable_colorFromSh.r);
+        // g = sRGB_to_linear(variable_colorFromSh.g);
+        // b = sRGB_to_linear(variable_colorFromSh.b);
+        // material.baseColor.rgb = vec3(r, g, b);
+    }
+}

cpp/open3d/visualization/shader/glsl/ShCoeffsToColorFast.glsl

@@ -0,0 +1,42 @@
+// Refering to [this link](https://github.com/nerfstudio-project/gsplat/blob/bd64a47414e182dc105c1a2fdb6691068518d060/gsplat/cuda/include/spherical_harmonics.cuh)
+
+vec3 sh0_coeffs_to_color_fast(vec3 coeffs_sh0) {
+    return 0.2820947917738781 * coeffs_sh0;
+}
+
+// vec3 dir must be normalized vector
+vec3 sh1_coeffs_to_color_fast(vec3 dir, float coeffs[3 * 3]) {
+    vec3 result;
+    for (int i = 0; i < 3; i++) {
+        result[i] = 0.48860251190292 * (-dir.y * coeffs[0 + i] + dir.z * coeffs[1 * 3 + i] - dir.x * coeffs[2 * 3 + i]);
+    }
+    return result;
+}
+
+// vec3 dir must be normalized vector
+vec3 sh2_coeffs_to_color_fast(vec3 dir, float coeffs[5 * 3]) {
+    float inorm = inversesqrt(dot(dir, dir));
+    float x = dir.x * inorm;
+    float y = dir.y * inorm;
+    float z = dir.z * inorm;
+
+    float z2 = dir.z * dir.z;
+    float fTmp0B = -1.092548430592079 * dir.z;
+    float fC1 = dir.x * dir.x - dir.y * dir.y;
+    float fS1 = 2.0 * dir.x * dir.y;
+
+    float pSH6 = (0.9461746957575601 * z2 - 0.3153915652525201);
+    float pSH7 = fTmp0B * dir.x;
+    float pSH5 = fTmp0B * dir.y;
+    float pSH8 = 0.5462742152960395 * fC1;
+    float pSH4 = 0.5462742152960395 * fS1;
+
+    vec3 result;
+    for (int i = 0; i < 3; i++) {
+        result[i] = pSH4 * coeffs[0 + i] + pSH5 * coeffs[1 * 3 + i] +
+        pSH6 * coeffs[2 * 3 + i] + pSH7 * coeffs[3 * 3 + i] +
+        pSH8 * coeffs[4 * 3 + i];
+    }
+
+    return result;
+}