Skip to content

Commit 001d279

Browse files
dli7319dli7319
authored
Merge pull request #123 from dli7319/depth_mesh
Use depth camera transform for depth mesh.
2 parents 54e0899 + 1cb0525 commit 001d279

File tree

3 files changed

+67
-80
lines changed

3 files changed

+67
-80
lines changed

‎src/depth/Depth.ts‎

Lines changed: 48 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,6 @@ export class Depth {
2121
// The main camera.
2222
private camera!: THREE.Camera;
2323
private renderer!: THREE.WebGLRenderer;
24-
private scene!: THREE.Scene;
25-
private projectionMatrixInverse = new THREE.Matrix4();
2624

2725
enabled = false;
2826
view: XRView[] = [];
@@ -50,8 +48,11 @@ export class Depth {
5048
private depthClients = new Set<object>();
5149

5250
depthProjectionMatrices: THREE.Matrix4[] = [];
51+
depthProjectionInverseMatrices: THREE.Matrix4[] = [];
5352
depthViewMatrices: THREE.Matrix4[] = [];
5453
depthViewProjectionMatrices: THREE.Matrix4[] = [];
54+
depthCameraPositions: THREE.Vector3[] = [];
55+
depthCameraRotations: THREE.Quaternion[] = [];
5556

5657
/**
5758
* Depth is a lightweight manager based on three.js to simply prototyping
@@ -77,7 +78,6 @@ export class Depth {
7778
this.camera = camera;
7879
this.options = options;
7980
this.renderer = renderer;
80-
this.scene = scene;
8181
this.enabled = options.enabled;
8282

8383
if (this.options.depthTexture.enabled) {
@@ -97,8 +97,7 @@ export class Depth {
9797
this.renderer.shadowMap.enabled = true;
9898
this.renderer.shadowMap.type = THREE.PCFShadowMap;
9999
}
100-
camera.add(this.depthMesh);
101-
scene.add(camera);
100+
scene.add(this.depthMesh);
102101
}
103102

104103
if (this.options.occlusion.enabled) {
@@ -123,24 +122,24 @@ export class Depth {
123122
}
124123

125124
/**
126-
* Projects the given world position to clip space and then to view
127-
* space using the depth.
125+
* Projects the given world position to depth camera's clip space and then
126+
* to the depth camera's view space using the depth.
128127
* @param position - The world position to project.
128+
* @returns The depth camera view space position.
129129
*/
130130
getProjectedDepthViewPositionFromWorldPosition(
131131
position: THREE.Vector3,
132132
target = new THREE.Vector3()
133133
) {
134-
const camera = this.renderer.xr?.getCamera?.()?.cameras?.[0] || this.camera;
135134
clipSpacePosition
136135
.copy(position)
137-
.applyMatrix4(camera.matrixWorldInverse)
138-
.applyMatrix4(camera.projectionMatrix);
136+
.applyMatrix4(this.depthViewMatrices[0])
137+
.applyMatrix4(this.depthProjectionMatrices[0]);
139138
const u = 0.5 * (clipSpacePosition.x + 1.0);
140139
const v = 0.5 * (clipSpacePosition.y + 1.0);
141140
const depth = this.getDepth(u, v);
142141
target.set(2.0 * (u - 0.5), 2.0 * (v - 0.5), -1);
143-
target.applyMatrix4(camera.projectionMatrixInverse);
142+
target.applyMatrix4(this.depthProjectionInverseMatrices[0]);
144143
target.multiplyScalar((target.z - depth) / target.z);
145144
return target;
146145
}
@@ -165,7 +164,7 @@ export class Depth {
165164
2.0 * (v - 0.5),
166165
-1
167166
);
168-
vertexPosition.applyMatrix4(this.projectionMatrixInverse);
167+
vertexPosition.applyMatrix4(this.depthProjectionInverseMatrices[0]);
169168
vertexPosition.multiplyScalar(-depth / vertexPosition.z);
170169
return vertexPosition;
171170
}
@@ -176,6 +175,9 @@ export class Depth {
176175
this.depthViewMatrices.push(new THREE.Matrix4());
177176
this.depthViewProjectionMatrices.push(new THREE.Matrix4());
178177
this.depthProjectionMatrices.push(new THREE.Matrix4());
178+
this.depthProjectionInverseMatrices.push(new THREE.Matrix4());
179+
this.depthCameraPositions.push(new THREE.Vector3());
180+
this.depthCameraRotations.push(new THREE.Quaternion());
179181
}
180182
if (depthData.projectionMatrix && depthData.transform) {
181183
this.depthProjectionMatrices[viewId].fromArray(
@@ -184,12 +186,28 @@ export class Depth {
184186
this.depthViewMatrices[viewId].fromArray(
185187
depthData.transform.inverse.matrix
186188
);
189+
this.depthCameraPositions[viewId].set(
190+
depthData.transform.position.x,
191+
depthData.transform.position.y,
192+
depthData.transform.position.z
193+
);
194+
this.depthCameraRotations[viewId].set(
195+
depthData.transform.orientation.x,
196+
depthData.transform.orientation.y,
197+
depthData.transform.orientation.z,
198+
depthData.transform.orientation.w
199+
);
187200
} else {
188201
const camera =
189202
this.renderer.xr?.getCamera()?.cameras?.[viewId] ?? this.camera;
190203
this.depthProjectionMatrices[viewId].copy(camera.projectionMatrix);
191204
this.depthViewMatrices[viewId].copy(camera.matrixWorldInverse);
205+
this.depthCameraPositions[viewId].copy(camera.position);
206+
this.depthCameraRotations[viewId].copy(camera.quaternion);
192207
}
208+
this.depthProjectionInverseMatrices[viewId]
209+
.copy(this.depthProjectionMatrices[viewId])
210+
.invert();
193211
this.depthViewProjectionMatrices[viewId].multiplyMatrices(
194212
this.depthProjectionMatrices[viewId],
195213
this.depthViewMatrices[viewId]
@@ -198,6 +216,7 @@ export class Depth {
198216

199217
updateCPUDepthData(depthData: XRCPUDepthInformation, viewId = 0) {
200218
this.cpuDepthData[viewId] = depthData;
219+
this.updateDepthMatrices(depthData, viewId);
201220

202221
// Updates Depth Array.
203222
if (this.depthArray[viewId] == null) {
@@ -221,14 +240,18 @@ export class Depth {
221240
}
222241

223242
if (this.options.depthMesh.enabled && this.depthMesh && viewId == 0) {
224-
this.depthMesh.updateDepth(depthData);
243+
this.depthMesh.updateDepth(
244+
depthData,
245+
this.depthProjectionInverseMatrices[0]
246+
);
247+
this.depthMesh.position.copy(this.depthCameraPositions[0]);
248+
this.depthMesh.quaternion.copy(this.depthCameraRotations[0]);
225249
}
226-
227-
this.updateDepthMatrices(depthData, viewId);
228250
}
229251

230252
updateGPUDepthData(depthData: XRWebGLDepthInformation, viewId = 0) {
231253
this.gpuDepthData[viewId] = depthData;
254+
this.updateDepthMatrices(depthData, viewId);
232255

233256
// For now, assume that we need cpu depth only if depth mesh is enabled.
234257
// In the future, add a separate option.
@@ -261,13 +284,19 @@ export class Depth {
261284

262285
if (this.options.depthMesh.enabled && this.depthMesh && viewId == 0) {
263286
if (cpuDepth) {
264-
this.depthMesh.updateDepth(cpuDepth);
287+
this.depthMesh.updateDepth(
288+
cpuDepth,
289+
this.depthProjectionInverseMatrices[0]
290+
);
265291
} else {
266-
this.depthMesh.updateGPUDepth(depthData);
292+
this.depthMesh.updateGPUDepth(
293+
depthData,
294+
this.depthProjectionInverseMatrices[0]
295+
);
267296
}
297+
this.depthMesh.position.copy(this.depthCameraPositions[0]);
298+
this.depthMesh.quaternion.copy(this.depthCameraRotations[0]);
268299
}
269-
270-
this.updateDepthMatrices(depthData, viewId);
271300
}
272301

273302
getTexture(viewId: number) {
@@ -286,12 +315,6 @@ export class Depth {
286315
}
287316

288317
updateLocalDepth(frame: XRFrame) {
289-
const leftCamera = this.renderer.xr?.getCamera?.()?.cameras?.[0];
290-
if (leftCamera && this.depthMesh && this.depthMesh.parent != leftCamera) {
291-
leftCamera.add(this.depthMesh);
292-
this.scene.add(leftCamera);
293-
}
294-
295318
const session = frame.session;
296319
const binding = this.renderer.xr.getBinding();
297320

‎src/depth/DepthMesh.ts‎

Lines changed: 19 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@ import {DepthTextures} from './DepthTextures';
1010

1111
export class DepthMesh extends MeshScript {
1212
static dependencies = {
13-
camera: THREE.Camera,
1413
renderer: THREE.WebGLRenderer,
1514
};
1615
static isDepthMesh = true;
@@ -32,8 +31,8 @@ export class DepthMesh extends MeshScript {
3231
private colliderUpdateFps: number;
3332

3433
private renderer!: THREE.WebGLRenderer;
35-
private camera!: THREE.Camera;
36-
private projectionMatrixInverse = new THREE.Matrix4();
34+
private projectionMatrixInverse: Readonly<THREE.Matrix4> =
35+
new THREE.Matrix4();
3736
private lastColliderUpdateTime = 0;
3837
private options: DepthMeshOptions;
3938
private depthTextureMaterialUniforms?;
@@ -90,7 +89,6 @@ export class DepthMesh extends MeshScript {
9089

9190
this.visible = options.showDebugTexture || options.renderShadow;
9291
this.options = options;
93-
this.projectionMatrixInverse = new THREE.Matrix4();
9492
this.lastColliderUpdateTime = performance.now();
9593
this.updateVertexNormals = options.updateVertexNormals;
9694
this.colliderUpdateFps = options.colliderUpdateFps;
@@ -112,27 +110,19 @@ export class DepthMesh extends MeshScript {
112110
/**
113111
* Initialize the depth mesh.
114112
*/
115-
init({
116-
camera,
117-
renderer,
118-
}: {
119-
camera: THREE.Camera;
120-
renderer: THREE.WebGLRenderer;
121-
}) {
122-
this.camera = camera;
113+
init({renderer}: {renderer: THREE.WebGLRenderer}) {
123114
this.renderer = renderer;
124115
}
125116

126117
/**
127118
* Updates the depth data and geometry positions based on the provided camera
128119
* and depth data.
129120
*/
130-
updateDepth(depthData: XRCPUDepthInformation) {
131-
const camera = this.renderer.xr?.getCamera?.()?.cameras?.[0] || this.camera;
132-
if (!camera) return;
133-
134-
// Inverts the projection matrix.
135-
this.projectionMatrixInverse.copy(camera.projectionMatrix).invert();
121+
updateDepth(
122+
depthData: Readonly<XRCPUDepthInformation>,
123+
projectionMatrixInverse: Readonly<THREE.Matrix4>
124+
) {
125+
this.projectionMatrixInverse = projectionMatrixInverse;
136126

137127
this.minDepth = 8;
138128
this.maxDepth = 0;
@@ -175,11 +165,14 @@ export class DepthMesh extends MeshScript {
175165
this.updateColliderIfNeeded();
176166
}
177167

178-
updateGPUDepth(depthData: XRWebGLDepthInformation) {
179-
this.updateDepth(this.convertGPUToGPU(depthData));
168+
updateGPUDepth(
169+
depthData: Readonly<XRWebGLDepthInformation>,
170+
projectionMatrixInverse: Readonly<THREE.Matrix4>
171+
) {
172+
this.updateDepth(this.convertGPUToGPU(depthData), projectionMatrixInverse);
180173
}
181174

182-
convertGPUToGPU(depthData: XRWebGLDepthInformation) {
175+
convertGPUToGPU(depthData: Readonly<XRWebGLDepthInformation>) {
183176
if (!this.depthTarget) {
184177
this.depthTarget = new THREE.WebGLRenderTarget(
185178
depthData.width,
@@ -338,7 +331,7 @@ export class DepthMesh extends MeshScript {
338331
/**
339332
* Optimizes collider updates to run periodically based on the specified FPS.
340333
*/
341-
updateColliderIfNeeded() {
334+
private updateColliderIfNeeded() {
342335
const timeSinceLastUpdate = performance.now() - this.lastColliderUpdateTime;
343336
if (this.RAPIER && timeSinceLastUpdate > 1000 / this.colliderUpdateFps) {
344337
this.getWorldPosition(this.worldPosition);
@@ -412,40 +405,16 @@ export class DepthMesh extends MeshScript {
412405
this.lastColliderUpdateTime = performance.now();
413406
}
414407

415-
getDepth(
416-
raycaster: THREE.Raycaster,
417-
ndc: THREE.Vector2,
418-
camera: THREE.Camera
419-
) {
420-
// Convert the point from blendedWorld space to normalized device
421-
// coordinates (NDC) const ndc = point.clone().project(camera);
422-
423-
// Create a Vector2 for the NDC x, y coordinates (used by the Raycaster)
424-
const ndc2D = new THREE.Vector2(ndc.x, ndc.y);
425-
426-
// Set up the Raycaster to cast a ray from the camera through the NDC point
427-
raycaster.setFromCamera(ndc2D, camera);
428-
429-
// Check for intersections with the mesh
430-
const intersects = raycaster.intersectObject(this);
431-
432-
// If an intersection is found, calculate the distance from the point to the
433-
// intersection
434-
if (intersects.length > 0) {
435-
const distance = intersects[0].distance;
436-
return distance;
437-
} else {
438-
return -1;
439-
}
440-
}
441-
442408
/**
443409
* Customizes raycasting to compute normals for intersections.
444410
* @param raycaster - The raycaster object.
445411
* @param intersects - Array to store intersections.
446412
* @returns - True if intersections are found.
447413
*/
448-
raycast(raycaster: THREE.Raycaster, intersects: THREE.Intersection[]) {
414+
override raycast(
415+
raycaster: THREE.Raycaster,
416+
intersects: THREE.Intersection[]
417+
) {
449418
const intersections: THREE.Intersection[] = [];
450419
if (this.downsampledMesh) {
451420
this.downsampledMesh.raycast(raycaster, intersections);

‎src/simulator/Simulator.ts‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ import {XREffects} from '../core/components/XREffects';
88
import {Options} from '../core/Options';
99
import {Script} from '../core/Script';
1010
import {Depth} from '../depth/Depth';
11-
import {DepthMesh} from '../depth/DepthMesh';
1211
import {Input} from '../input/Input';
1312

1413
import {SimulatorCamera} from './SimulatorCamera';
@@ -112,7 +111,6 @@ export class Simulator extends Script {
112111
if (this.initialized) return;
113112
// Get optional dependencies from the registry.
114113
const deviceCamera = registry.get(XRDeviceCamera);
115-
const depthMesh = registry.get(DepthMesh);
116114
this.options = simulatorOptions;
117115
camera.position.copy(this.options.initialCameraPosition);
118116
this.userInterface.init(simulatorOptions, this.controls, this.hands);
@@ -130,9 +128,6 @@ export class Simulator extends Script {
130128
if (options.depth.enabled) {
131129
this.renderDepthPass = true;
132130
this.depth.init(renderer, camera, depth);
133-
if (options.depth.depthMesh.enabled && depthMesh) {
134-
camera.add(depthMesh);
135-
}
136131
}
137132
scene.add(camera);
138133

0 commit comments

Comments
 (0)