Open
Description
Checklist
- I have searched for similar issues.
- For Python issues, I have tested with the latest development wheel.
- I have checked the release documentation and the latest documentation (for
masterbranch).
Describe the issue
I have been noticing that on my Mac (MacBook Pro (14-inch, 2021), macOS version 12.6.7 (21G651)) that the mouse_and_point_coord.py example does not work properly. Points returned when clicking on the image are incorrect. By modifying the example (see below), I was able to trace this to an incorrect depth image (which seems to have an incorrect camera setup) returned by self.widget3d.scene.scene.render_to_depth_image(depth_callback). Calling self.widget3d.scene.scene.render_to_image(image_callback) on the other hand returns a correct image.
Steps to reproduce the bug
# ----------------------------------------------------------------------------
# - Open3D: www.open3d.org -
# ----------------------------------------------------------------------------
# Copyright (c) 2018-2023 www.open3d.org
# SPDX-License-Identifier: MIT
# ----------------------------------------------------------------------------
import matplotlib.pyplot as plt
import numpy as np
import open3d as o3d
import open3d.visualization.gui as gui
import open3d.visualization.rendering as rendering
# This example displays a point cloud and if you Ctrl-click on a point
# (Cmd-click on macOS) it will show the coordinates of the point.
# This example illustrates:
# - custom mouse handling on SceneWidget
# - getting a the depth value of a point (OpenGL depth)
# - converting from a window point + OpenGL depth to world coordinate
class ExampleApp:
def __init__(self, cloud):
# We will create a SceneWidget that fills the entire window, and then
# a label in the lower left on top of the SceneWidget to display the
# coordinate.
app = gui.Application.instance
self.window = app.create_window("Open3D - GetCoord Example", 1024, 768)
# Since we want the label on top of the scene, we cannot use a layout,
# so we need to manually layout the window's children.
self.window.set_on_layout(self._on_layout)
self.widget3d = gui.SceneWidget()
self.window.add_child(self.widget3d)
self.info = gui.Label("")
self.info.visible = False
self.window.add_child(self.info)
self.widget3d.scene = rendering.Open3DScene(self.window.renderer)
mat = rendering.MaterialRecord()
mat.shader = "defaultUnlit"
# Point size is in native pixels, but "pixel" means different things to
# different platforms (macOS, in particular), so multiply by Window scale
# factor.
mat.point_size = 3 * self.window.scaling
self.widget3d.scene.add_geometry("Point Cloud", cloud, mat)
bounds = self.widget3d.scene.bounding_box
center = bounds.get_center()
self.widget3d.setup_camera(60, bounds, center)
self.widget3d.look_at(center, center - [0, 0, 3], [0, -1, 0])
self.widget3d.set_on_mouse(self._on_mouse_widget3d)
def _on_layout(self, layout_context):
r = self.window.content_rect
self.widget3d.frame = r
pref = self.info.calc_preferred_size(layout_context,
gui.Widget.Constraints())
self.info.frame = gui.Rect(r.x,
r.get_bottom() - pref.height, pref.width,
pref.height)
def _on_mouse_widget3d(self, event):
# We could override BUTTON_DOWN without a modifier, but that would
# interfere with manipulating the scene.
if event.type == gui.MouseEvent.Type.BUTTON_DOWN and event.is_modifier_down(
gui.KeyModifier.CTRL):
def depth_callback(depth_image):
# Coordinates are expressed in absolute coordinates of the
# window, but to dereference the image correctly we need them
# relative to the origin of the widget. Note that even if the
# scene widget is the only thing in the window, if a menubar
# exists it also takes up space in the window (except on macOS).
x = event.x - self.widget3d.frame.x
y = event.y - self.widget3d.frame.y
depth_array = np.asarray(depth_image)
print(f"Frame: {self.widget3d.frame.width}x{self.widget3d.frame.height}")
print(f"Array shape: {depth_array.shape}")
plt.imsave("depth_buffer.png", depth_array)
# Note that np.asarray() reverses the axes.
depth = depth_array[y, x]
if depth == 1.0: # clicked on nothing (i.e. the far plane)
text = ""
else:
world = self.widget3d.scene.camera.unproject(
x, y, depth, self.widget3d.frame.width,
self.widget3d.frame.height)
text = "({:.3f}, {:.3f}, {:.3f})".format(
world[0], world[1], world[2])
# This is not called on the main thread, so we need to
# post to the main thread to safely access UI items.
def update_label():
self.info.text = text
self.info.visible = (text != "")
# We are sizing the info label to be exactly the right size,
# so since the text likely changed width, we need to
# re-layout to set the new frame.
self.window.set_needs_layout()
gui.Application.instance.post_to_main_thread(
self.window, update_label)
self.widget3d.scene.scene.render_to_depth_image(depth_callback)
def image_callback(image):
plt.imsave("image.png", np.asarray(image))
self.widget3d.scene.scene.render_to_image(image_callback)
return gui.Widget.EventCallbackResult.HANDLED
return gui.Widget.EventCallbackResult.IGNORED
def main():
app = gui.Application.instance
app.initialize()
# This example will also work with a triangle mesh, or any 3D object.
# If you use a triangle mesh you will probably want to set the material
# shader to "defaultLit" instead of "defaultUnlit".
pcd_data = o3d.data.DemoICPPointClouds()
cloud = o3d.io.read_point_cloud(pcd_data.paths[0])
ex = ExampleApp(cloud)
app.run()
if __name__ == "__main__":
main()Error message
No error message.
Screen shot of window.
Depth image saved upon mouse click.
Image saved upon mouse click.
Expected behavior
Expect the depth image to align with the color image.
Open3D, Python and System information
- Operating system: macOS 12.6.7 (21G651)
- Python version: 3.9.16 (main, Jul 17 2023, 19:26:31) [Clang 14.0.0 (clang-1400.0.29.202)
- Open3D version: output from python: 0.17.0
- System architecture: apple-silicon
- Is this a remote workstation?: no
- How did you install Open3D?: pipAdditional information
No response