perf(viewport): render camera frustums with GPU instancing (#1312)

* perf(viewport): render camera frustums with GPU instancing

Camera frustum overlays were rebuilt and re-uploaded on the CPU every frame for every visible camera (~160k vertices/frame for large COLMAP datasets), causing viewport lag while navigating with frustums enabled. The cost was CPU/driver-bound, so it was most noticeable on Windows.

Perspective frustums now draw via a single instanced draw per viewport panel: one storage-buffer instance per camera (model matrix + color), with the vertex shader generating the 8 edges, projecting them with the same camera-space pinhole as the scene, and expanding them into anti-aliased, depth-faded lines (reusing the shape-overlay fragment logic). Thumbnails and equirectangular frustums keep the existing path.

Co-Authored-By: Oz <oz-agent@warp.dev>

* fix(viewport): handle pano frustum instancing

* refactor(viewport): trim frustum instancing path

---------

Co-authored-by: Janusch Patas <janusch.patas@gmx.de>

Author: jacobvanbeets

src/visualizer/CMakeLists.txt

@@ -291,6 +291,7 @@ compile_shader(lfs_visualizer "${LFS_VIEWPORT_SHADER_SOURCE_DIR}/textured_overla
 compile_shader(lfs_visualizer "${LFS_VIEWPORT_SHADER_SOURCE_DIR}/textured_overlay.frag" "${LFS_VIEWPORT_SHADER_BINARY_DIR}/textured_overlay.frag.spv.h" kTexturedOverlayFragSpv)
 compile_shader(lfs_visualizer "${LFS_VIEWPORT_SHADER_SOURCE_DIR}/pivot.vert" "${LFS_VIEWPORT_SHADER_BINARY_DIR}/pivot.vert.spv.h" kPivotVertSpv)
 compile_shader(lfs_visualizer "${LFS_VIEWPORT_SHADER_SOURCE_DIR}/pivot.frag" "${LFS_VIEWPORT_SHADER_BINARY_DIR}/pivot.frag.spv.h" kPivotFragSpv)
+compile_shader(lfs_visualizer "${LFS_VIEWPORT_SHADER_SOURCE_DIR}/frustum.vert" "${LFS_VIEWPORT_SHADER_BINARY_DIR}/frustum.vert.spv.h" kFrustumVertSpv)
 compile_shader(lfs_visualizer "${LFS_VIEWPORT_SHADER_SOURCE_DIR}/mesh.vert" "${LFS_VIEWPORT_SHADER_BINARY_DIR}/mesh.vert.spv.h" kMeshVertSpv)
 compile_shader(lfs_visualizer "${LFS_VIEWPORT_SHADER_SOURCE_DIR}/mesh.frag" "${LFS_VIEWPORT_SHADER_BINARY_DIR}/mesh.frag.spv.h" kMeshFragSpv)
 compile_shader(lfs_visualizer "${LFS_VIEWPORT_SHADER_SOURCE_DIR}/mesh_shadow.vert" "${LFS_VIEWPORT_SHADER_BINARY_DIR}/mesh_shadow.vert.spv.h" kMeshShadowVertSpv)

src/visualizer/gui/gui_manager.cpp

@@ -973,57 +973,6 @@ namespace lfs::vis::gui {
             };
         }
 
-        [[nodiscard]] std::optional<glm::vec2> projectPointToPanelScreen(
-            const VulkanGuidePanelTarget& panel,
-            const RenderSettings& settings,
-            const glm::vec3& world) {
-            const glm::mat3 rotation = panel.viewport->getRotationMatrix();
-            const glm::vec3 translation = panel.viewport->getTranslation();
-            const glm::vec3 view = glm::transpose(rotation) * (world - translation);
-
-            if (settings.equirectangular) {
-                const float len = glm::length(view);
-                if (!std::isfinite(len) || len <= 1e-6f) {
-                    return std::nullopt;
-                }
-                const glm::vec3 dir = view / len;
-                const float ndc_x = std::atan2(dir.x, -dir.z) / glm::pi<float>();
-                const float ndc_y = -std::asin(std::clamp(dir.y, -1.0f, 1.0f)) /
-                                    (glm::pi<float>() * 0.5f);
-                if (!std::isfinite(ndc_x) || !std::isfinite(ndc_y)) {
-                    return std::nullopt;
-                }
-                return panel.pos + glm::vec2((ndc_x * 0.5f + 0.5f) * panel.size.x,
-                                             (ndc_y * 0.5f + 0.5f) * panel.size.y);
-            }
-
-            constexpr float kMinViewZ = -1e-4f;
-            if (view.z >= kMinViewZ) {
-                return std::nullopt;
-            }
-
-            const float width = static_cast<float>(std::max(panel.render_size.x, 1));
-            const float height = static_cast<float>(std::max(panel.render_size.y, 1));
-            const float cx = width * 0.5f;
-            const float cy = height * 0.5f;
-            if (settings.orthographic) {
-                if (!std::isfinite(settings.ortho_scale) || settings.ortho_scale <= 0.0f) {
-                    return std::nullopt;
-                }
-                return renderToPanelScreen(panel, glm::vec2(cx + view.x * settings.ortho_scale,
-                                                            cy - view.y * settings.ortho_scale));
-            }
-
-            const auto [fx, fy] = lfs::rendering::computePixelFocalLengths(
-                panel.render_size, settings.focal_length_mm);
-            const float depth = -view.z;
-            if (depth <= 0.0f) {
-                return std::nullopt;
-            }
-            return renderToPanelScreen(panel, glm::vec2(cx + view.x * fx / depth,
-                                                        cy - view.y * fy / depth));
-        }
-
         [[nodiscard]] bool projectedQuadVisible(const std::array<glm::vec2, 4>& points,
                                                 const VulkanGuidePanelTarget& panel) {
             glm::vec2 min_point(std::numeric_limits<float>::max());
@@ -2068,46 +2017,6 @@ namespace lfs::vis::gui {
             return visualizer_camera_to_world * fov_scale;
         }
 
-        void appendPerspectiveCameraFrustum(VulkanViewportPassParams& params,
-                                            const VulkanGuidePanelTarget& panel,
-                                            const RenderSettings& settings,
-                                            const glm::mat4& model,
-                                            const glm::vec4& color) {
-            constexpr std::array local_points{
-                glm::vec3(-0.5f, -0.5f, -1.0f),
-                glm::vec3(0.5f, -0.5f, -1.0f),
-                glm::vec3(0.5f, 0.5f, -1.0f),
-                glm::vec3(-0.5f, 0.5f, -1.0f),
-                glm::vec3(0.0f, 0.0f, 0.0f),
-            };
-            constexpr std::array<std::pair<int, int>, 8> edges{{
-                {0, 1},
-                {1, 2},
-                {2, 3},
-                {3, 0},
-                {0, 4},
-                {1, 4},
-                {2, 4},
-                {3, 4},
-            }};
-
-            std::array<glm::vec3, local_points.size()> world_points{};
-            for (size_t i = 0; i < local_points.size(); ++i) {
-                world_points[i] = glm::vec3(model * glm::vec4(local_points[i], 1.0f));
-            }
-            for (const auto& [a, b] : edges) {
-                addProjectedOverlayLine(params.overlay_triangles,
-                                        params,
-                                        panel,
-                                        settings,
-                                        world_points[static_cast<size_t>(a)],
-                                        world_points[static_cast<size_t>(b)],
-                                        color,
-                                        1.5f,
-                                        true);
-            }
-        }
-
         void appendEquirectangularCameraFrustum(VulkanViewportPassParams& params,
                                                 const VulkanGuidePanelTarget& panel,
                                                 const RenderSettings& settings,
@@ -2218,9 +2127,58 @@ namespace lfs::vis::gui {
             }
 
             constexpr float kMinRenderAlpha = 0.01f;
-            const glm::vec3 view_position = panel.viewport->getTranslation();
+            const glm::mat3 panel_rotation = panel.viewport->getRotationMatrix();
+            const glm::vec3 panel_translation = panel.viewport->getTranslation();
+            const glm::mat3 world_to_panel_rotation = glm::transpose(panel_rotation);
+            const glm::vec3 view_position = panel_translation;
+            const float panel_render_width = static_cast<float>(std::max(panel.render_size.x, 1));
+            const float panel_render_height = static_cast<float>(std::max(panel.render_size.y, 1));
+            const float panel_cx = panel_render_width * 0.5f;
+            const float panel_cy = panel_render_height * 0.5f;
+            const auto [panel_fx, panel_fy] =
+                lfs::rendering::computePixelFocalLengths(panel.render_size, settings.focal_length_mm);
+            const auto project_panel_point = [&](const glm::vec3& world) -> std::optional<glm::vec2> {
+                const glm::vec3 view = world_to_panel_rotation * (world - panel_translation);
+                if (settings.equirectangular) {
+                    const float len = glm::length(view);
+                    if (!std::isfinite(len) || len <= 1e-6f) {
+                        return std::nullopt;
+                    }
+                    const glm::vec3 dir = view / len;
+                    const float ndc_x = std::atan2(dir.x, -dir.z) / glm::pi<float>();
+                    const float ndc_y = -std::asin(std::clamp(dir.y, -1.0f, 1.0f)) /
+                                        (glm::pi<float>() * 0.5f);
+                    if (!std::isfinite(ndc_x) || !std::isfinite(ndc_y)) {
+                        return std::nullopt;
+                    }
+                    return panel.pos + glm::vec2((ndc_x * 0.5f + 0.5f) * panel.size.x,
+                                                 (ndc_y * 0.5f + 0.5f) * panel.size.y);
+                }
+
+                constexpr float kMinViewZ = -1e-4f;
+                if (view.z >= kMinViewZ) {
+                    return std::nullopt;
+                }
+                if (settings.orthographic) {
+                    if (!std::isfinite(settings.ortho_scale) || settings.ortho_scale <= 0.0f) {
+                        return std::nullopt;
+                    }
+                    return renderToPanelScreen(panel, glm::vec2(panel_cx + view.x * settings.ortho_scale,
+                                                                panel_cy - view.y * settings.ortho_scale));
+                }
+
+                const float depth = -view.z;
+                if (depth <= 0.0f) {
+                    return std::nullopt;
+                }
+                return renderToPanelScreen(panel, glm::vec2(panel_cx + view.x * panel_fx / depth,
+                                                            panel_cy - view.y * panel_fy / depth));
+            };
             size_t background_thumbnail_requests = 0;
             constexpr size_t kBackgroundThumbnailRequestsPerFrame = 16;
+            const std::uint32_t frustum_first_instance =
+                static_cast<std::uint32_t>(params.frustum_instances.size());
+            params.frustum_instances.reserve(params.frustum_instances.size() + cameras.size());
             for (size_t i = 0; i < cameras.size(); ++i) {
                 const auto& camera = cameras[i];
                 if (!camera) {
@@ -2274,18 +2232,16 @@ namespace lfs::vis::gui {
                     std::array<glm::vec2, image_corners.size()> screen_points{};
                     std::array<float, image_corners.size()> corner_depths{};
                     bool quad_visible = true;
-                    const glm::mat3 panel_rotation = panel.viewport->getRotationMatrix();
-                    const glm::vec3 panel_translation = panel.viewport->getTranslation();
                     for (size_t corner = 0; corner < image_corners.size(); ++corner) {
                         const glm::vec3 world_point =
                             glm::vec3((*model) * glm::vec4(image_corners[corner], 1.0f));
-                        const auto projected = projectPointToPanelScreen(panel, settings, world_point);
+                        const auto projected = project_panel_point(world_point);
                         if (!projected) {
                             quad_visible = false;
                             break;
                         }
                         screen_points[corner] = *projected;
-                        const glm::vec3 view = glm::transpose(panel_rotation) * (world_point - panel_translation);
+                        const glm::vec3 view = world_to_panel_rotation * (world_point - panel_translation);
                         corner_depths[corner] = settings.equirectangular ? glm::length(view) : -view.z;
                     }
                     quad_visible = quad_visible && projectedQuadVisible(screen_points, panel);
@@ -2310,10 +2266,32 @@ namespace lfs::vis::gui {
                                                   {emphasis_mix, disabled_mix, 0.0f, 0.0f},
                                                   corner_depths);
                     }
-                    appendPerspectiveCameraFrustum(params, panel, settings, *model, color);
+                    params.frustum_instances.push_back(VulkanViewportFrustumInstance{
+                        .model = *model,
+                        .color = color,
+                    });
                 }
             }
 
+            const std::uint32_t frustum_instance_count =
+                static_cast<std::uint32_t>(params.frustum_instances.size()) - frustum_first_instance;
+            if (frustum_instance_count > 0) {
+                const glm::mat4 frustum_view =
+                    lfs::rendering::makeViewMatrix(panel_rotation, panel_translation);
+                params.frustum_batches.push_back(VulkanViewportFrustumBatch{
+                    .view = frustum_view,
+                    .viewport_pos = panel.pos,
+                    .viewport_size = panel.size,
+                    .render_size = glm::vec2(panel.render_size),
+                    .focal_x = settings.orthographic ? settings.ortho_scale : panel_fx,
+                    .focal_y = settings.orthographic ? settings.ortho_scale : panel_fy,
+                    .orthographic = settings.orthographic,
+                    .equirectangular = settings.equirectangular,
+                    .first_instance = frustum_first_instance,
+                    .instance_count = frustum_instance_count,
+                });
+            }
+
             if (thumbnail_cache.hasPendingWork()) {
                 rendering_manager.markDirty(DirtyFlag::OVERLAY);
             }