[WebRTC] Refactor texture handling in ASSIMP export and improve data channel reliability

Author: ssheorey

cpp/open3d/t/io/file_format/FileASSIMP.cpp

@@ -566,97 +566,97 @@ bool WriteTriangleMeshUsingASSIMP(const std::string& filename,
             ai_mat->AddProperty(&ac, 1, AI_MATKEY_COLOR_EMISSIVE);
         }
 
-        // Count texture maps...
+        // Build a list of texture-embed actions in a single pass. Each action
+        // is a lambda that writes one texture slot into the assimp scene; the
+        // slot index is passed at call time. .
+        //
         // NOTE: GLTF2 expects a single combined roughness/metal map. If the
         // model has one we just export it, otherwise if both roughness and
         // metal maps are available we combine them, otherwise if only one or
         // the other is available we just export the one map.
         const auto& material = w_mesh.GetMaterial();
-        int n_textures = 0;
-        if (HasValidTexture(material, "albedo")) ++n_textures;
-        if (HasValidTexture(material, "normal")) ++n_textures;
-        if (HasValidTexture(material, "ambient_occlusion")) ++n_textures;
+        using TextureAction = std::function<void(int /*slot*/)>;
+        std::vector<TextureAction> texture_actions;
+
+        if (HasValidTexture(material, "albedo")) {
+            texture_actions.push_back([&](int idx) {
+                auto img = material.GetAlbedoMap();
+                SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx,
+                                           aiTextureType_DIFFUSE, img);
+                SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx,
+                                           aiTextureType_BASE_COLOR, img);
+            });
+        }
+        if (HasValidTexture(material, "ambient_occlusion")) {
+            texture_actions.push_back([&](int idx) {
+                auto img = material.GetAOMap();
+                SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx,
+                                           aiTextureType_LIGHTMAP, img);
+            });
+        }
         if (HasValidTexture(material, "ao_rough_metal")) {
-            ++n_textures;
+            texture_actions.push_back([&](int idx) {
+                auto img = material.GetAORoughnessMetalMap();
+                SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx,
+                                           aiTextureType_UNKNOWN, img);
+            });
         } else if (HasValidTexture(material, "roughness") &&
                    HasValidTexture(material, "metallic")) {
-            ++n_textures;
+            texture_actions.push_back([&](int idx) {
+                auto rough = material.GetRoughnessMap().AsTensor();
+                auto metal = material.GetMetallicMap().AsTensor();
+                if (rough.GetShape() != metal.GetShape()) {
+                    utility::LogError(
+                            "RoughnessMap (shape={}) and MetallicMap "
+                            "(shape={}) must have the same shape.",
+                            rough.GetShape(), metal.GetShape());
+                }
+                auto rows = rough.GetShape(0);
+                auto cols = rough.GetShape(1);
+                auto rough_metal = core::Tensor::Full({rows, cols, 4}, 255,
+                                                      core::Dtype::UInt8);
+                rough_metal.Slice(2, 2, 3) =
+                        metal.Slice(2, 0, 1);  // blue channel is metal
+                rough_metal.Slice(2, 1, 2) =
+                        rough.Slice(2, 0, 1);  // green channel is roughness
+                geometry::Image rough_metal_img(rough_metal);
+                SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx,
+                                           aiTextureType_UNKNOWN,
+                                           rough_metal_img);
+            });
         } else {
-            if (HasValidTexture(material, "roughness")) ++n_textures;
-            if (HasValidTexture(material, "metallic")) ++n_textures;
+            if (HasValidTexture(material, "roughness")) {
+                texture_actions.push_back([&](int idx) {
+                    auto img = material.GetRoughnessMap();
+                    SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx,
+                                               aiTextureType_UNKNOWN, img);
+                });
+            }
+            if (HasValidTexture(material, "metallic")) {
+                texture_actions.push_back([&](int idx) {
+                    auto img = material.GetMetallicMap();
+                    SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx,
+                                               aiTextureType_UNKNOWN, img);
+                });
+            }
         }
+        if (HasValidTexture(material, "normal")) {
+            texture_actions.push_back([&](int idx) {
+                auto img = material.GetNormalMap();
+                SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx,
+                                           aiTextureType_NORMALS, img);
+            });
+        }
+
+        int n_textures = static_cast<int>(texture_actions.size());
         if (n_textures > 0) {
             ai_scene->mTextures = new aiTexture*[n_textures];
             for (int i = 0; i < n_textures; ++i) {
                 ai_scene->mTextures[i] = new aiTexture();
+                texture_actions[i](i);
             }
             ai_scene->mNumTextures = n_textures;
         }
-
-        // Now embed the textures that are available...
-        int current_idx = 0;
-        if (HasValidTexture(material, "albedo")) {
-            auto img = material.GetAlbedoMap();
-            SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx,
-                                       aiTextureType_DIFFUSE, img);
-            SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx,
-                                       aiTextureType_BASE_COLOR, img);
-            ++current_idx;
-        }
-        if (HasValidTexture(w_mesh.GetMaterial(), "ambient_occlusion")) {
-            auto img = w_mesh.GetMaterial().GetAOMap();
-            SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx,
-                                       aiTextureType_LIGHTMAP, img);
-            ++current_idx;
-        }
-        if (HasValidTexture(w_mesh.GetMaterial(), "ao_rough_metal")) {
-            auto img = w_mesh.GetMaterial().GetAORoughnessMetalMap();
-            SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx,
-                                       aiTextureType_UNKNOWN, img);
-            ++current_idx;
-        } else if (HasValidTexture(w_mesh.GetMaterial(), "roughness") &&
-                   HasValidTexture(w_mesh.GetMaterial(), "metallic")) {
-            auto rough = w_mesh.GetMaterial().GetRoughnessMap().AsTensor();
-            auto metal = w_mesh.GetMaterial().GetMetallicMap().AsTensor();
-            if (rough.GetShape() != metal.GetShape()) {
-                utility::LogError(
-                        "RoughnessMap (shape={}) and MetallicMap (shape={}) "
-                        "must have the same shape.",
-                        rough.GetShape(), metal.GetShape());
-            }
-            auto rows = rough.GetShape(0);
-            auto cols = rough.GetShape(1);
-            auto rough_metal = core::Tensor::Full({rows, cols, 4}, 255,
-                                                  core::Dtype::UInt8);
-            rough_metal.Slice(2, 2, 3) =
-                    metal.Slice(2, 0, 1);  // blue channel is metal
-            rough_metal.Slice(2, 1, 2) =
-                    rough.Slice(2, 0, 1);  // green channel is roughness
-
-            geometry::Image rough_metal_img(rough_metal);
-            SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx,
-                                       aiTextureType_UNKNOWN, rough_metal_img);
-            ++current_idx;
-        } else {
-            if (HasValidTexture(material, "roughness")) {
-                auto img = material.GetRoughnessMap();
-                SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx,
-                                           aiTextureType_UNKNOWN, img);
-                ++current_idx;
-            }
-            if (HasValidTexture(material, "metallic")) {
-                auto img = material.GetMetallicMap();
-                SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx,
-                                           aiTextureType_UNKNOWN, img);
-                ++current_idx;
-            }
-        }
-        if (HasValidTexture(material, "normal")) {
-            auto img = material.GetNormalMap();
-            SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx,
-                                       aiTextureType_NORMALS, img);
-            ++current_idx;
-        }
     }
     ai_scene->mMaterials[0] = ai_mat;
 

cpp/open3d/visualization/webrtc_server/WebRTCWindowSystem.cpp

@@ -364,10 +364,8 @@ std::string WebRTCWindowSystem::OnDataChannelMessage(
         if (impl_->data_channel_message_callbacks_.count(class_name) != 0) {
             reply = impl_->data_channel_message_callbacks_.at(class_name)(
                     message);
-            // Do not call PostRedrawEvent here. Mouse/keyboard callbacks
-            // already schedule a redraw via Window::OnMouseEvent → PostRedraw.
-            // An extra PostRedrawEvent here creates a duplicate draw event
-            // before the input event is even processed, causing backlog.
+            // Custom callbacks that mutate GUI state (e.g. add/remove geometry)
+            // must call window->PostRedraw() or post_redraw() themselves.
             return reply;
         } else {
             reply = fmt::format(

cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js

@@ -699,16 +699,15 @@ let WebRtcStreamer = (function() {
         };
 
         // Local datachannel sends data.
-        // Use unordered, unreliable delivery for mouse/input events. Ordered
-        // reliable delivery causes head-of-line blocking: a lost packet stalls
-        // all subsequent events until it is retransmitted. For interactive
-        // mouse events the latest position is all that matters, so a dropped
-        // message is better than a delayed one. maxRetransmits: 0 means the
-        // browser sends once and moves on; ordered: false removes sequencing.
+        // Use reliable ordered delivery (the default). While unordered +
+        // unreliable would reduce head-of-line blocking for mouse MOVE/DRAG,
+        // the same channel carries discrete events (BUTTON_DOWN/UP, key events,
+        // resize) and application-level RPC (tensorboard/update_geometry etc.)
+        // that must not be lost or reordered. Mouse coalescing (rAF + server-
+        // side replace_or_merge_mouse) already prevents event backlog, so the
+        // extra reliability is free in practice on a local network.
         try {
-            this.dataChannel = pc.createDataChannel(
-                    'ClientDataChannel',
-                    {ordered: false, maxRetransmits: 0});
+            this.dataChannel = pc.createDataChannel('ClientDataChannel');
             var dataChannel = this.dataChannel;
             dataChannel.onopen = function() {
                 console.log('local datachannel open');