Fix Kit+Newton recording gap: skip Kit Replicator for Newton physics

Newton's Fabric transform writes (via wp.fabricarray Warp kernel) do not
trigger RTX's scene delegate change notifications, so Replicator annotator
buffers stay zeroed regardless of how many app.update() calls are issued.

Fix _select_video_backend to detect Newton physics (video_capture_backend()
== "newton_gl") and skip Kit Replicator in that case. If a Newton GL
visualizer is configured, use its framebuffer directly. If only a Kit
visualizer is configured alongside Newton, log a warning and fall back to
standalone Newton GL capture, which renders Newton's scene correctly.

Kit recording continues to work unchanged with PhysX.

Add two unit tests for the new fallback dispatch and remove the xfail from
test_kit_visualizer_newton_source_records_rgb — all 5 integration tests now
pass (previously 4 passed, 1 xfailed).

Author: matthewtrepte

source/isaaclab/changelog.d/mtrepte-refactor-viz-and-recording-cfg-classes.minor.rst

@@ -23,3 +23,8 @@ Changed
 * Removed ``eye`` and ``lookat`` fields from
   :class:`~isaaclab.envs.utils.VideoRecorderCfg`. The recorder is now passive: it records
   whatever the active visualizer or physics backend renders without repositioning the camera.
+
+* When Newton is the active physics backend, :func:`~isaaclab.envs.utils.video_recorder._select_video_backend`
+  now skips Kit Replicator capture (Newton Fabric writes do not notify RTX's scene delegate,
+  producing black frames) and falls back to standalone Newton GL capture with a logged warning.
+  Kit recording continues to work with PhysX.

source/isaaclab/isaaclab/envs/utils/video_recorder.py

@@ -47,10 +47,32 @@ def _select_video_backend(scene: InteractiveScene, backend_source: str) -> tuple
 
     if backend_source == "visualizer":
         visualizer_cfgs = scene.sim._resolve_visualizer_cfgs()
-        for visualizer_type, backend in (("kit", "kit"), ("newton", "newton_gl")):
+        physics_backend = scene.sim.physics_manager.video_capture_backend()
+
+        if physics_backend != "newton_gl":
+            # PhysX: Kit Replicator and Newton GL visualizers both supported.
+            for visualizer_type, backend in (("kit", "kit"), ("newton", "newton_gl")):
+                for visualizer_cfg in visualizer_cfgs:
+                    if visualizer_cfg.visualizer_type == visualizer_type:
+                        return backend, visualizer_cfg
+        else:
+            # Newton physics: Kit Replicator recording does not work because
+            # Newton's Fabric transform writes do not trigger RTX's scene
+            # delegate — the annotator buffer stays black. Use Newton GL
+            # capture instead (from an active Newton visualizer if present,
+            # or standalone otherwise).
             for visualizer_cfg in visualizer_cfgs:
-                if visualizer_cfg.visualizer_type == visualizer_type:
-                    return backend, visualizer_cfg
+                if visualizer_cfg.visualizer_type == "newton":
+                    return "newton_gl", visualizer_cfg
+            kit_requested = any(getattr(cfg, "visualizer_type", None) == "kit" for cfg in visualizer_cfgs)
+            if kit_requested:
+                import logging
+
+                logging.getLogger(__name__).warning(
+                    "Kit Replicator recording is not supported with Newton physics "
+                    "(Newton Fabric writes do not notify the RTX scene delegate). "
+                    "Falling back to standalone Newton GL capture."
+                )
 
     backend = scene.sim.physics_manager.video_capture_backend()
     if backend is not None:

source/isaaclab/test/envs/test_video_recorder.py

@@ -80,6 +80,28 @@ def test_resolve_backend_selects_newton_visualizer():
     assert visualizer_cfg is newton_cfg
 
 
+def test_kit_visualizer_falls_back_to_newton_gl_when_newton_physics():
+    """Kit visualizer is skipped for Newton physics; falls back to Newton GL standalone."""
+    kit_cfg = _make_visualizer_cfg("kit")
+    scene = _make_scene([kit_cfg], video_backend="newton_gl")
+
+    backend, visualizer_cfg = _select_video_backend(scene, "visualizer")
+
+    assert backend == "newton_gl"
+    assert visualizer_cfg is None
+
+
+def test_kit_visualizer_is_used_when_physx_physics():
+    """Kit visualizer is still preferred when PhysX is the physics backend."""
+    kit_cfg = _make_visualizer_cfg("kit")
+    scene = _make_scene([kit_cfg], video_backend="kit")
+
+    backend, visualizer_cfg = _select_video_backend(scene, "visualizer")
+
+    assert backend == "kit"
+    assert visualizer_cfg is kit_cfg
+
+
 def test_resolve_backend_renderer_source_ignores_visualizers():
     """Renderer source bypasses active visualizers and falls through to physics manager."""
     newton_cfg = _make_visualizer_cfg("newton")

source/isaaclab_tasks/test/core/test_recording_backends.py

@@ -7,34 +7,29 @@
 
 ``_select_video_backend`` has three dispatch paths:
 
-1. ``"kit"`` visualizer present    → Kit Replicator capture (IsaacsimKitPerspectiveVideo)
-2. ``"newton"`` visualizer present → Newton GL framebuffer  (NewtonGLVisualizer.render_rgb_array)
-3. Neither above (any other visualizer or no visualizer)
+1. ``"kit"`` visualizer present + PhysX → Kit Replicator capture (IsaacsimKitPerspectiveVideo)
+2. ``"newton"`` visualizer present      → Newton GL framebuffer  (NewtonGLVisualizer.render_rgb_array)
+3. Neither above, OR Kit visualizer with Newton physics
    → physics manager video_capture_backend():
      "kit"        → standalone Kit capture
      "newton_gl"  → standalone Newton GL capture
      None         → RuntimeError (not tested here; covered by test_video_recorder.py)
 
+When Newton is the physics backend, Kit Replicator recording is skipped
+(Newton Fabric writes do not notify RTX's scene delegate, producing black frames).
+_select_video_backend falls back to Newton GL standalone with a logged warning.
+
 Path 3 fallthrough (Rerun, Viser, NewtonRTX visualizer types) is covered at the
 unit level by test_video_recorder.py::test_resolve_backend_unsupported_visualizer_falls_back.
-NewtonRTXVisualizerCfg is not yet a factory-registered type and cannot be
-instantiated as a visualizer, so it has no integration test here.
-
-Known limitation (xfail): Kit Replicator recording does not produce frames when
-Newton is the physics backend. Kit recording relies on
-``ensure_isaac_rtx_render_update`` called by the PhysX manager's render
-callback; when Newton is active that callback is absent, so the render product
-buffer stays zeroed even though the Kit visualizer pumps ``app.update()``.
-Tracked as a known gap — not a regression introduced by this re-arch.
 
 Setup:
     - AppLauncher(headless=True, enable_cameras=True)
     - CartpoleEnv (DirectRLEnv, 1 env) as the test vehicle.
 Tests:
     - KitVisualizerCfg + PhysX + backend_source="visualizer"
-      -> Kit Replicator capture (path 1, PhysX physics) [passes]
+      -> Kit Replicator capture (path 1) [passes]
     - KitVisualizerCfg + Newton + backend_source="visualizer"
-      -> Kit Replicator capture (path 1, Newton physics) [xfail: see above]
+      -> Newton GL standalone (path 3 fallback: Kit skipped for Newton physics) [passes]
     - NewtonGLVisualizerCfg + Newton + backend_source="visualizer"
       -> Newton GL framebuffer capture (path 2) [passes]
     - PhysX + backend_source="renderer"
@@ -130,17 +125,13 @@ def test_kit_visualizer_physx_source_records_rgb():
     _assert_valid_frame(frame, "kit-visualizer-physx")
 
 
-@pytest.mark.xfail(
-    strict=True,
-    reason=(
-        "Kit Replicator recording produces a zero buffer when Newton is the physics backend. "
-        "The PhysX manager registers an ensure_isaac_rtx_render_update render callback that "
-        "primes the render product; Newton does not, so the buffer stays zeroed even though "
-        "the Kit visualizer pumps app.update(). Known gap — not a regression of this re-arch."
-    ),
-)
 def test_kit_visualizer_newton_source_records_rgb():
-    """Kit visualizer + Newton physics → Kit Replicator capture stays all-black (known limitation)."""
+    """Kit visualizer + Newton + backend_source='visualizer' → Newton GL standalone (Kit skipped for Newton).
+
+    Kit Replicator recording does not work with Newton physics because Newton Fabric
+    writes do not notify RTX's scene delegate. _select_video_backend detects Newton
+    and falls back to standalone Newton GL capture with a logged warning.
+    """
     from isaaclab_newton.physics import MJWarpSolverCfg, NewtonCfg
     from isaaclab_visualizers.kit import KitVisualizerCfg
 
@@ -158,13 +149,13 @@ def test_kit_visualizer_newton_source_records_rgb():
 
 
 def test_newton_gl_visualizer_source_records_rgb():
-    """NewtonGLVisualizerCfg + Newton + backend_source='visualizer' → Newton GL framebuffer capture."""
+    """NewtonVisualizerCfg + Newton + backend_source='visualizer' → Newton GL framebuffer capture."""
     from isaaclab_newton.physics import MJWarpSolverCfg, NewtonCfg
-    from isaaclab_visualizers.newton import NewtonGLVisualizerCfg
+    from isaaclab_visualizers.newton import NewtonVisualizerCfg
 
     env_cfg = _cartpole_cfg(backend_source="visualizer")
     env_cfg.sim.physics = NewtonCfg(solver_cfg=MJWarpSolverCfg())
-    env_cfg.sim.visualizer_cfgs = [NewtonGLVisualizerCfg(window_width=_W, window_height=_H)]
+    env_cfg.sim.visualizer_cfgs = [NewtonVisualizerCfg(window_width=_W, window_height=_H)]
 
     frame = _capture_frame(env_cfg)
     _assert_valid_frame(frame, "newton-gl-visualizer-source")