Fix camera render API.

Author: duburcqa

genesis/ext/pyrender/offscreen.py

@@ -148,7 +148,7 @@ def render(
         if shadow and not self._is_software:
             flags |= RenderFlags.SHADOWS_ALL
 
-        if depth and not rgb and not seg and not normal:
+        if depth and not (rgb or seg):
             flags |= RenderFlags.DEPTH_ONLY
 
         if plane_reflection and not self._is_software:
@@ -166,22 +166,25 @@ def render(
         if depth:
             flags |= RenderFlags.RET_DEPTH
 
-        if self._platform.supports_framebuffers():
-            flags |= RenderFlags.OFFSCREEN
-            retval = renderer.render(scene, flags, seg_node_map)
-        else:
-            if flags & RenderFlags.ENV_SEPARATE:
-                gs.raise_exception("'env_separate_rigid=True' not supported on this platform.")
-            if normal:
-                gs.raise_exception("'normal=True' not supported on this platform.")
-            renderer.render(scene, flags, seg_node_map)
-            if depth:
-                depth = renderer.read_depth_buf()
-            if flags & RenderFlags.DEPTH_ONLY:
-                retval = (depth,)
+        if rgb or depth or seg:
+            if self._platform.supports_framebuffers():
+                flags |= RenderFlags.OFFSCREEN
+                retval = renderer.render(scene, flags, seg_node_map)
             else:
-                color = renderer.read_color_buf()
-                retval = (color, depth)
+                if flags & RenderFlags.ENV_SEPARATE:
+                    gs.raise_exception("'env_separate_rigid=True' not supported on this platform.")
+                if normal:
+                    gs.raise_exception("'normal=True' not supported on this platform.")
+                renderer.render(scene, flags, seg_node_map)
+                if depth:
+                    depth = renderer.read_depth_buf()
+                if flags & RenderFlags.DEPTH_ONLY:
+                    retval = (depth,)
+                else:
+                    color = renderer.read_color_buf()
+                    retval = (color, depth)
+        else:
+            retval = ()
 
         if normal:
             class CustomShaderCache:
@@ -204,7 +207,7 @@ def get_program(self, vertex_shader, fragment_shader, geometry_shader=None, defi
             if env_separate_rigid:
                 flags |= RenderFlags.ENV_SEPARATE
             glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
-            normal_arr, _ = renderer.render(scene, flags, is_first_pass=False)
+            normal_arr, *_ = renderer.render(scene, flags, is_first_pass=False, force_skip_shadows=True)
             retval = (*retval, normal_arr)
 
             renderer._program_cache = old_cache

genesis/ext/pyrender/renderer.py

@@ -115,7 +115,7 @@ def point_size(self):
     def point_size(self, value):
         self._point_size = float(value)
 
-    def render(self, scene, flags, seg_node_map=None, is_first_pass=True):
+    def render(self, scene, flags, seg_node_map=None, *, is_first_pass=True, force_skip_shadows=False):
         """Render a scene with the given set of flags.
 
         Parameters
@@ -159,7 +159,7 @@ def render(self, scene, flags, seg_node_map=None, is_first_pass=True):
             env_idx = i if use_env_idx else -1
 
             # Render necessary shadow maps
-            if not (flags & RenderFlags.SEG or flags & RenderFlags.DEPTH_ONLY):
+            if not (force_skip_shadows or flags & RenderFlags.SEG or flags & RenderFlags.DEPTH_ONLY):
                 for ln in scene.light_nodes:
                     take_pass = False
                     if isinstance(ln.light, DirectionalLight) and flags & RenderFlags.SHADOWS_DIRECTIONAL:
@@ -873,7 +873,7 @@ def _configure_forward_pass_viewport(self, flags):
         # If using offscreen render, bind main framebuffer
         if flags & RenderFlags.OFFSCREEN:
             self._configure_main_framebuffer()
-            if (flags & RenderFlags.SEG or flags & RenderFlags.DEPTH_ONLY):
+            if flags & RenderFlags.SEG or flags & RenderFlags.DEPTH_ONLY:
                 glBindFramebuffer(GL_DRAW_FRAMEBUFFER, self._main_fb)
             else:
                 glBindFramebuffer(GL_DRAW_FRAMEBUFFER, self._main_fb_ms)

genesis/ext/pyrender/viewer.py

@@ -1138,10 +1138,13 @@ def _render(self, camera_node=None, renderer=None, normal=False):
 
         if self.render_flags["depth"]:
             flags |= RenderFlags.RET_DEPTH
-            if not (self.render_flags["rgb"] or self.render_flags["seg"] or normal):
+            if not (self.render_flags["rgb"] or self.render_flags["seg"]):
                 flags |= RenderFlags.DEPTH_ONLY
 
-        retval = renderer.render(self.scene, flags, seg_node_map=seg_node_map)
+        if self.render_flags["rgb"] or self.render_flags["depth"] or self.render_flags["seg"]:
+            retval = renderer.render(self.scene, flags, seg_node_map=seg_node_map)
+        else:
+            retval = ()
 
         if normal:
             class CustomShaderCache:
@@ -1164,8 +1167,8 @@ def get_program(self, vertex_shader, fragment_shader, geometry_shader=None, defi
             if self.render_flags["env_separate_rigid"]:
                 flags |= RenderFlags.ENV_SEPARATE
             glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
-            normal_arr, _ = renderer.render(scene, flags, is_first_pass=False)
-            retval = retval + (normal_arr,)
+            normal_arr, *_ = renderer.render(scene, flags, is_first_pass=False)
+            retval = (*retval, normal_arr)
 
             renderer._program_cache = old_cache
 

genesis/vis/camera.py

@@ -371,10 +371,6 @@ def render(
         normal_arr : np.ndarray
             The rendered surface normal(s).
         """
-
-        if (rgb or depth or segmentation or normal) is False:
-            gs.raise_exception("Nothing to render.")
-
         rgb_arr, depth_arr, seg_arr, seg_color_arr, normal_arr = None, None, None, None, None
 
         if self._batch_renderer is not None:
@@ -485,7 +481,7 @@ def render_pointcloud(self, world_frame=True):
         mask_arr : np.ndarray
             The valid depth mask.
         """
-        # Compute the (denormalized) depth map using PyRender systematically
+        # Compute the (denormalized) depth map using PyRender systematically.
         # TODO: Add support of BatchRendered (requires access to projection matrix)
         self._rasterizer.update_scene()
         rgb_arr, depth_arr, seg_idxc_arr, normal_arr = self._rasterizer.render_camera(

genesis/vis/rasterizer.py

@@ -84,7 +84,7 @@ def render_camera(self, camera, rgb=True, depth=False, segmentation=False, norma
                 )
 
             if segmentation:
-                seg_idxc_rgb_arr, _ = self._renderer.render(
+                seg_idxc_rgb_arr, *_ = self._renderer.render(
                     self._context._scene,
                     self._camera_targets[camera.uid],
                     camera_node=self._camera_nodes[camera.uid],
@@ -112,7 +112,7 @@ def render_camera(self, camera, rgb=True, depth=False, segmentation=False, norma
                 )
 
             if segmentation:
-                seg_idxc_rgb_arr, _ = self._viewer.render_offscreen(
+                seg_idxc_rgb_arr, *_ = self._viewer.render_offscreen(
                     self._camera_nodes[camera.uid],
                     self._camera_targets[camera.uid],
                     rgb=False,
@@ -129,7 +129,7 @@ def render_camera(self, camera, rgb=True, depth=False, segmentation=False, norma
         if depth:
             depth_arr = retval[int(rgb)]
         if normal:
-            normal_arr = retval[int(rgb + rgb_arr)]
+            normal_arr = retval[int(rgb + depth)]
         return rgb_arr, depth_arr, seg_idxc_arr, normal_arr
 
     def update_scene(self):

tests/test_render.py

@@ -1,3 +1,4 @@
+import itertools
 import queue
 import sys
 from io import BytesIO
@@ -246,7 +247,7 @@ def test_batched_offscreen_rendering(show_viewer, tol):
     )
     scene.build(n_envs=3, env_spacing=(2.0, 2.0))
 
-    for _ in range(10):
+    for _ in range(7):
         dofs_lower_bound, dofs_upper_bound = robot.get_dofs_limit()
         qpos = dofs_lower_bound + (dofs_upper_bound - dofs_lower_bound) * torch.rand(robot.n_qs)
 
@@ -271,6 +272,47 @@ def test_batched_offscreen_rendering(show_viewer, tol):
             assert_allclose(steps_rgb_arrays[0][i], steps_rgb_arrays[1][i], tol=tol)
 
 
+@pytest.mark.required
+def test_render_api(show_viewer):
+    scene = gs.Scene(
+        show_viewer=show_viewer,
+        show_FPS=False,
+    )
+    scene.add_entity(
+        morph=gs.morphs.Sphere(
+            pos=(0.0, 0.0, 0.0),
+            radius=1.0,
+            fixed=True,
+        ),
+    )
+    camera = scene.add_camera(
+        pos=(0.0, 0.0, 10.0),
+        lookat=(0.0, 0.0, 0.0),
+        GUI=show_viewer,
+    )
+    scene.build()
+
+    rgb_arrs, depth_arrs, seg_arrs, normal_arrs = [], [], [], []
+    for rgb, depth, seg, normal in itertools.product((True, False), repeat=4):
+        rgb_arr, depth_arr, seg_arr, normal_arr = camera.render(rgb=rgb, depth=depth, segmentation=seg, normal=normal)
+        if rgb:
+            rgb_arrs.append(rgb_arr.astype(np.float32))
+        if depth:
+            depth_arrs.append(depth_arr.astype(np.float32))
+        if seg:
+            seg_arrs.append(seg_arr.astype(np.float32))
+        if normal:
+            normal_arrs.append(normal_arr.astype(np.float32))
+
+    assert_allclose(np.diff(rgb_arrs, axis=0), 0.0, tol=gs.EPS)
+    assert_allclose(np.diff(seg_arrs, axis=0), 0.0, tol=gs.EPS)
+    assert_allclose(np.diff(normal_arrs, axis=0), 0.0, tol=gs.EPS)
+    # Depth is not matching at machine-precision because of MSAA being disabled for depth-only
+    # FIXME: There is one pixel off on MacOS with Apple's Software Rendering, probably due to a bug...
+    tol = 2e-4 if sys.platform == "darwin" else gs.EPS
+    assert_allclose(np.diff(depth_arrs, axis=0)[[0, 1, 2, 4, 5, 6]], 0.0, tol=tol)
+
+
 @pytest.mark.required
 def test_point_cloud(show_viewer):
     CAMERA_DIST = 8.0
@@ -312,7 +354,8 @@ def test_point_cloud(show_viewer):
         GUI=show_viewer,
     )
     for camera in scene.visualizer.cameras:
-        camera._near = 1.0
+        camera._near = 2.0
+        camera._far = 15.0
     scene.build()
 
     if show_viewer:
@@ -329,12 +372,14 @@ def test_point_cloud(show_viewer):
     point_cloud = point_cloud[mask]
     point_cloud = point_cloud @ gu.z_up_to_R(np.array((1.0, 1.0, 1.0)), np.array((0.0, 0.0, 1.0))).T
     point_cloud -= np.array((CAMERA_DIST, CAMERA_DIST, CAMERA_DIST))
-    assert_allclose(np.linalg.norm(point_cloud, ord=float("inf"), axis=-1), BOX_HALFSIZE, atol=1e-4)
+    # FIXME: Tolerance must be increased whe using Apple's Software Rendering, probably due to a bug...
+    tol = 2e-4 if sys.platform == "darwin" else 1e-4
+    assert_allclose(np.linalg.norm(point_cloud, ord=float("inf"), axis=-1), BOX_HALFSIZE, atol=tol)
 
     point_cloud, mask = camera_box_2.render_pointcloud(world_frame=True)
     point_cloud = point_cloud[mask]
     point_cloud += np.array((0.0, OBJ_OFFSET, 0.0))
-    assert_allclose(np.linalg.norm(point_cloud, ord=float("inf"), axis=-1), BOX_HALFSIZE, atol=1e-4)
+    assert_allclose(np.linalg.norm(point_cloud, ord=float("inf"), axis=-1), BOX_HALFSIZE, atol=tol)
 
     # It is not possible to get higher accuracy because of tesselation
     point_cloud, mask = camera_sphere.render_pointcloud(world_frame=False)
@@ -387,21 +432,19 @@ def test_batched_mounted_camera_rendering(show_viewer, tol):
     for cam in cams:
         cam.attach(robot.get_link("hand"), gu.trans_R_to_T(trans, R))
 
-    target_quat = np.tile(np.array([0, 1, 0, 0]), [n_envs, 1])  # pointing downwards
-    center = np.tile(np.array([-0.25, -0.25, 0.5]), [n_envs, 1])
+    target_quat = np.tile(np.array([0, 1, 0, 0]), (n_envs, 1))  # pointing downwards
+    center = np.tile(np.array([-0.25, -0.25, 0.5]), (n_envs, 1))
     rng = np.random.default_rng(42)
     angular_speed = rng.uniform(-10, 10, n_envs)
     r = 0.25
 
     ee_link = robot.get_link("hand")
 
     steps_rgb_queue: queue.Queue[list[np.ndarray]] = queue.Queue(maxsize=2)
-
-    for i in range(50):
-        target_pos = np.zeros([n_envs, 3])
-        target_pos[:, 0] = center[:, 0] + np.cos(i / 360 * np.pi * angular_speed) * r
-        target_pos[:, 1] = center[:, 1] + np.sin(i / 360 * np.pi * angular_speed) * r
-        target_pos[:, 2] = center[:, 2]
+    for i in range(20):
+        target_pos = center.copy()
+        target_pos[:, 0] += np.cos(i / 360 * np.pi * angular_speed) * r
+        target_pos[:, 1] += np.sin(i / 360 * np.pi * angular_speed) * r
 
         q = robot.inverse_kinematics(
             link=ee_link,