optimize the performance (fps) of mesh detector in xrblocks

Author: JFGVL1187

demos/ballpit/BallPit.js

@@ -1,6 +1,7 @@
 import * as THREE from 'three';
 import * as xb from 'xrblocks';
 import {palette} from 'xrblocks/addons/utils/Palette.js';
+import Stats from 'three/addons/libs/stats.module.js';
 
 import {BallShooter} from './BallShooter.js';
 
@@ -30,19 +31,70 @@ export class BallPit extends xb.Script {
     this.lastBallCreatedTimeForController = new Map();
     this.pointer = new THREE.Vector2();
     this.velocity = new THREE.Vector3();
+
+    // Initialize Stats for FPS dashboard
+    this.stats = new Stats();
+    this.stats.dom.style.width = '80px';
+    this.stats.dom.style.height = '48px';
+    this.stats.showPanel(0); // 0: fps, 1: ms, 2: mb
+    document.body.appendChild(this.stats.dom);
+
+    this.statsMesh = null;
   }
 
   init() {
     xb.add(this);
+    this.createStatsMesh();
   }
 
   update() {
     super.update();
+
+    // Update stats
+    this.stats.update();
+    if (this.statsMesh && this.statsMesh.material.map) {
+      this.statsMesh.material.map.needsUpdate = true;
+    }
+
+    // Update stats mesh position to follow camera in XR mode
+    if (this.statsMesh && xb.core.renderer.xr.isPresenting) {
+      const camera = xb.core.renderer.xr.getCamera();
+      if (camera && camera.cameras && camera.cameras.length > 0) {
+        const xrCamera = camera.cameras[0];
+        // Position stats mesh relative to camera
+        const offset = new THREE.Vector3(-0.15, 0.15, -0.5);
+        offset.applyQuaternion(xrCamera.quaternion);
+        this.statsMesh.position.copy(xrCamera.position).add(offset);
+        this.statsMesh.quaternion.copy(xrCamera.quaternion);
+        this.statsMesh.rotateY(Math.PI / 6);
+      }
+    }
+
     for (const controller of xb.core.input.controllers) {
       this.controllerUpdate(controller);
     }
   }
 
+  /**
+   * Creates a 3D mesh to display the FPS stats in XR space.
+   */
+  createStatsMesh() {
+    const statsCanvas = this.stats.dom.children[0];
+    const geometry = new THREE.PlaneGeometry(0.25, 0.15);
+    const texture = new THREE.CanvasTexture(statsCanvas);
+    const material = new THREE.MeshBasicMaterial({
+      map: texture,
+      transparent: true,
+      opacity: 0.9,
+      side: THREE.DoubleSide,
+    });
+    this.statsMesh = new THREE.Mesh(geometry, material);
+
+    this.statsMesh.position.set(-0.15, 0.15, -0.5);
+
+    this.add(this.statsMesh);
+  }
+
   // Adds hemisphere light for ambient lighting and directional light.
   addLights() {
     this.add(new THREE.HemisphereLight(0xbbbbbb, 0x888888, 3));

src/world/mesh/DetectedMesh.ts

@@ -60,4 +60,16 @@ export class DetectedMesh extends THREE.Mesh {
       );
     }
   }
+
+  dispose() {
+    if (this.blendedWorld && this.collider) {
+      this.blendedWorld.removeCollider(this.collider, false);
+      this.collider = undefined;
+    }
+    if (this.blendedWorld && this.rigidBody) {
+      this.blendedWorld.removeRigidBody(this.rigidBody);
+      this.rigidBody = undefined;
+    }
+    this.geometry.dispose();
+  }
 }

src/world/mesh/MeshDetector.ts

@@ -5,8 +5,8 @@ import {DetectedMesh} from './DetectedMesh';
 import {MeshDetectionOptions} from './MeshDetectionOptions';
 import {Physics} from '../../physics/Physics';
 
-const SEMANTIC_LABELS = ['Floor', 'Ceiling', 'Wall', 'Table'];
-const SEMANTIC_COLORS = [0x00ff00, 0xff0000, 0x0000ff, 0xffff00];
+const SEMANTIC_LABELS = ['floor', 'ceiling', 'wall'];
+const SEMANTIC_COLORS = [0x00ff00, 0xffff00, 0x0000ff];
 
 // Wrapper around WebXR Mesh Detection API
 // https://immersive-web.github.io/real-world-meshing/
@@ -23,6 +23,23 @@ export class MeshDetector extends Script {
   private physics?: Physics;
   private defaultMaterial = new THREE.MeshBasicMaterial({visible: false});
 
+  // Optimization1: Camera culling constants
+  private readonly kMaxViewDistance = 3.0;
+  private readonly kFOVCosThreshold = 0.25;
+
+  // Optimization2: Limit the number of meshes processed per frame (Not used)
+  private readonly MAX_MESHES_PER_FRAME = 100000; // Process only n meshes per frame
+
+  // Optimization3: Mesh update throttling (similar to ARCore reflection cube map in /usr/local/google/home/adamren/Desktop/xrlabs/arlabs/xrblocks/samples/lighting)
+  private lastMeshUpdateTime = 0;
+  private readonly MESH_UPDATE_INTERVAL_MS = 1000;
+
+  // Optimization4: Cleanup old meshes (ToDo: Not used)
+  private meshLastSeenTime = new Map<XRMesh, number>();
+  private readonly MAX_MESH_COUNT = 200; // Limit total mesh count
+  private readonly MESH_CLEANUP_INTERVAL_MS = 5000; // Cleanup every 5 seconds
+  private lastCleanupTime = 0;
+
   override init({
     options,
     renderer,
@@ -62,19 +79,66 @@ export class MeshDetector extends Script {
     const meshes = frame?.detectedMeshes;
     if (!meshes) return;
 
+    // Throttle mesh updates to ~30fps while rendering continues at full rate
+    const now = performance.now();
+    const timeSinceLastUpdate = now - this.lastMeshUpdateTime;
+
+    if (timeSinceLastUpdate < this.MESH_UPDATE_INTERVAL_MS) {
+      // Skip mesh update this frame - rendering continues without blocking
+      return;
+    }
+
+    this.lastMeshUpdateTime = now;
+
+    const referenceSpace = this.renderer.xr.getReferenceSpace();
+    if (!referenceSpace) return;
+    const {position: cameraPosition, forward: cameraForward} =
+      this.getCameraInfo(frame, referenceSpace);
+
     // Delete old meshes
     for (const [xrMesh, threeMesh] of this.xrMeshToThreeMesh.entries()) {
       if (!meshes.has(xrMesh)) {
         this.xrMeshToThreeMesh.delete(xrMesh);
         this.threeMeshToXrMesh.delete(threeMesh);
-        threeMesh.geometry.dispose();
+        threeMesh.dispose();
         this.remove(threeMesh);
       }
     }
 
-    // Add new meshes
+    // Limit processing to avoid frame drops
+    let processedCount = 0;
+    // const limitedMeshes = Array.from(meshes).slice(0, this.MAX_MESHES_PER_FRAME);
+    // const testMeshes = new Set(limitedMeshes);
+    // Process meshes with camera culling BEFORE creating/updating them
     for (const xrMesh of meshes) {
+      if (processedCount >= this.MAX_MESHES_PER_FRAME) break;
+
+      // Camera culling: only process visible meshes
+      if (
+        !this.shouldShowMeshInView(
+          xrMesh,
+          cameraPosition,
+          cameraForward,
+          frame,
+          referenceSpace
+        )
+      ) {
+        // If mesh exists but is not visible, remove it for performance
+        if (this.xrMeshToThreeMesh.has(xrMesh)) {
+          const threeMesh = this.xrMeshToThreeMesh.get(xrMesh)!;
+          this.xrMeshToThreeMesh.delete(xrMesh);
+          this.threeMeshToXrMesh.delete(threeMesh);
+          threeMesh.dispose();
+          this.remove(threeMesh);
+        }
+        continue; // Skip this mesh - don't create or update it
+      }
+
+      processedCount++;
+
+      // Only process meshes that pass camera culling
       if (!this.xrMeshToThreeMesh.has(xrMesh)) {
+        // New mesh - create it
         const threeMesh = this.createMesh(frame, xrMesh);
         this.xrMeshToThreeMesh.set(xrMesh, threeMesh);
         this.threeMeshToXrMesh.set(threeMesh, xrMesh);
@@ -86,6 +150,7 @@ export class MeshDetector extends Script {
           );
         }
       } else {
+        // Existing mesh - update vertices and pose
         const threeMesh = this.xrMeshToThreeMesh.get(xrMesh)!;
         threeMesh.updateVertices(xrMesh);
         this.updateMeshPose(frame, xrMesh, threeMesh);
@@ -114,4 +179,83 @@ export class MeshDetector extends Script {
       mesh.quaternion.copy(pose.transform.orientation);
     }
   }
+
+  /**
+   * Gets camera position and forward vector from XR frame.
+   */
+  private getCameraInfo(
+    frame: XRFrame,
+    referenceSpace: XRReferenceSpace
+  ): {
+    position: THREE.Vector3;
+    forward: THREE.Vector3;
+  } {
+    const viewerPose = frame.getViewerPose(referenceSpace);
+    const cameraPosition = new THREE.Vector3(0, 0, 0);
+    let cameraForward = new THREE.Vector3(0, 0, -1);
+
+    if (viewerPose && viewerPose.views && viewerPose.views.length > 0) {
+      // Get camera position from first view's transform
+      const viewTransform = viewerPose.views[0].transform;
+      const viewMatrix = new THREE.Matrix4().fromArray(viewTransform.matrix);
+      cameraPosition.setFromMatrixPosition(viewMatrix);
+
+      // Extract forward vector from matrix (typically -Z axis)
+      const forward = new THREE.Vector3(0, 0, -1);
+      forward.applyMatrix4(viewMatrix);
+      forward.sub(cameraPosition).normalize();
+      cameraForward = forward;
+    }
+
+    return {position: cameraPosition, forward: cameraForward};
+  }
+
+  /**
+   * Checks if mesh should be visible based on camera position and FOV.
+   */
+  private shouldShowMeshInView(
+    mesh: XRMesh,
+    cameraPosition: THREE.Vector3,
+    cameraForward: THREE.Vector3,
+    frame: XRFrame,
+    referenceSpace: XRReferenceSpace
+  ): boolean {
+    const meshPose = frame.getPose(mesh.meshSpace, referenceSpace);
+    if (!meshPose) {
+      return true; // Default to showing if no pose available
+    }
+
+    const meshMatrix = new THREE.Matrix4().fromArray(meshPose.transform.matrix);
+    const meshPosition = new THREE.Vector3();
+    meshPosition.setFromMatrixPosition(meshMatrix);
+
+    // Calculate distance
+    const dx = meshPosition.x - cameraPosition.x;
+    const dy = meshPosition.y - cameraPosition.y;
+    const dz = meshPosition.z - cameraPosition.z;
+    const distance = Math.sqrt(dx * dx + dy * dy + dz * dz);
+
+    if (distance > this.kMaxViewDistance) {
+      return false;
+    }
+
+    // Calculate direction vector and dot product (FOV check)
+    if (distance > 0.001) {
+      const invDistance = 1.0 / distance;
+      const dirX = dx * invDistance;
+      const dirY = dy * invDistance;
+      const dirZ = dz * invDistance;
+
+      const dotForward =
+        dirX * cameraForward.x +
+        dirY * cameraForward.y +
+        dirZ * cameraForward.z;
+
+      if (dotForward < this.kFOVCosThreshold) {
+        return false;
+      }
+    }
+
+    return true;
+  }
 }