Add native OpenCV lens-distortion cameras for OVRTX

CONTRIBUTORS.md

@@ -125,6 +125,7 @@ Guidelines for modifications:
 * Krishna Lakhi
 * Lin He
 * Lionel Gulich
+* Lior Ben Horin
 * Lorenz Wellhausen
 * Lotus Li
 * Louis Le Lay

docs/source/api/lab/isaaclab.sim.spawners.rst

@@ -211,6 +211,9 @@ Sensors
 
     PinholeCameraCfg
     FisheyeCameraCfg
+    OpenCvDistortionCfg
+    OpenCvPinholeDistortionCfg
+    OpenCvFisheyeDistortionCfg
 
 .. autofunction:: spawn_camera
 
@@ -222,6 +225,18 @@ Sensors
     :members:
     :exclude-members: __init__, func
 
+.. autoclass:: OpenCvDistortionCfg
+    :members:
+    :exclude-members: __init__, func
+
+.. autoclass:: OpenCvPinholeDistortionCfg
+    :members:
+    :exclude-members: __init__, func
+
+.. autoclass:: OpenCvFisheyeDistortionCfg
+    :members:
+    :exclude-members: __init__, func
+
 From Files
 ----------
 

source/isaaclab/changelog.d/opencv-lens-distortion-cameras.minor.rst

@@ -0,0 +1,17 @@
+Added
+^^^^^
+
+* Added :class:`~isaaclab.sim.spawners.sensors.OpenCvPinholeDistortionCfg` and
+  :class:`~isaaclab.sim.spawners.sensors.OpenCvFisheyeDistortionCfg` and a ``distortion`` field on
+  :class:`~isaaclab.sim.spawners.sensors.PinholeCameraCfg`, letting a camera carry an OpenCV
+  ``fx/fy/cx/cy`` + distortion-coefficient calibration. The model is authored on the camera prim as
+  the ``omni:lensdistortion:*`` USD API, which the RTX/OVRTX renderer honors natively.
+
+Changed
+^^^^^^^
+
+* Changed :meth:`~isaaclab.sensors.camera.Camera._update_intrinsic_matrices` to read the authored
+  ``fx/fy/cx/cy`` when an OpenCV lens-distortion model is present, so
+  :attr:`~isaaclab.sensors.camera.Camera.data` intrinsics reflect a real calibration (non-square
+  pixels or an off-center principal point) instead of assuming ``fx == fy`` and a centered principal
+  point.

source/isaaclab/isaaclab/sensors/camera/camera.py

@@ -204,6 +204,8 @@ def __init__(self, cfg: CameraCfg):
 
         # UsdGeom Camera prim for the sensor
         self._sensor_prims: list[UsdGeom.Camera] = list()
+        # Guard so the lens-distortion image-size mismatch warning is emitted at most once.
+        self._warned_distortion_image_size: bool = False
         # Allocated in :meth:`_create_buffers` once the renderer's output contract is known.
         self._data: CameraData | None = None
         # Renderer and render data — assigned in _initialize_impl.
@@ -651,18 +653,43 @@ def _update_intrinsic_matrices(self, env_ids: Sequence[int] | wp.array | None =
         for matrix_id, i in enumerate(env_ids_np):
             # Get corresponding sensor prim
             sensor_prim = self._sensor_prims[int(i)]
-            # get camera parameters
-            # currently rendering does not use aperture offsets or vertical aperture
-            focal_length = sensor_prim.GetFocalLengthAttr().Get()
-            horiz_aperture = sensor_prim.GetHorizontalApertureAttr().Get()
-
             # get viewport parameters
             height, width = self.image_shape
-            # extract intrinsic parameters
-            f_x = (width * focal_length) / horiz_aperture
-            f_y = f_x
-            c_x = width * 0.5
-            c_y = height * 0.5
+            # Prefer an authored OpenCV lens-distortion model when present: it carries the
+            # authoritative fx/fy/cx/cy that the RTX/OVRTX renderer projects through. These may be
+            # non-square (fx != fy) or off-center, unlike the focal-length/aperture readback which
+            # assumes square pixels and a centered principal point.
+            raw_prim = sensor_prim.GetPrim()
+            distortion_model = raw_prim.GetAttribute("omni:lensdistortion:model").Get()
+            if distortion_model:
+                prefix = f"omni:lensdistortion:{distortion_model}"
+                f_x = float(raw_prim.GetAttribute(f"{prefix}:fx").Get())
+                f_y = float(raw_prim.GetAttribute(f"{prefix}:fy").Get())
+                c_x = float(raw_prim.GetAttribute(f"{prefix}:cx").Get())
+                c_y = float(raw_prim.GetAttribute(f"{prefix}:cy").Get())
+                # the intrinsics are reported in the calibrated pixel space; warn once if the render
+                # resolution differs, since the reported matrix will not match the rendered pixels.
+                image_size = raw_prim.GetAttribute(f"{prefix}:imageSize").Get()
+                if image_size is not None and (int(image_size[0]), int(image_size[1])) != (width, height):
+                    if not self._warned_distortion_image_size:
+                        logger.warning(
+                            "Camera lens-distortion 'imageSize' %s does not match the render resolution"
+                            " (%d, %d). The reported intrinsic matrix is in the calibrated pixel space.",
+                            (int(image_size[0]), int(image_size[1])),
+                            width,
+                            height,
+                        )
+                        self._warned_distortion_image_size = True
+            else:
+                # get camera parameters
+                # currently rendering does not use aperture offsets or vertical aperture
+                focal_length = sensor_prim.GetFocalLengthAttr().Get()
+                horiz_aperture = sensor_prim.GetHorizontalApertureAttr().Get()
+                # extract intrinsic parameters
+                f_x = (width * focal_length) / horiz_aperture
+                f_y = f_x
+                c_x = width * 0.5
+                c_y = height * 0.5
             # create intrinsic matrix for depth linear
             intrinsic_matrices[matrix_id, 0, 0] = f_x
             intrinsic_matrices[matrix_id, 0, 2] = c_x

source/isaaclab/isaaclab/sim/__init__.pyi

@@ -137,6 +137,9 @@ __all__ = [
     "spawn_camera",
     "spawn_sensor_frame",
     "FisheyeCameraCfg",
+    "OpenCvDistortionCfg",
+    "OpenCvFisheyeDistortionCfg",
+    "OpenCvPinholeDistortionCfg",
     "PinholeCameraCfg",
     "SensorFrameCfg",
     "spawn_capsule",
@@ -332,6 +335,9 @@ from .spawners import (
     MjcfFileCfg,
     MultiAssetSpawnerCfg,
     MultiUsdFileCfg,
+    OpenCvDistortionCfg,
+    OpenCvFisheyeDistortionCfg,
+    OpenCvPinholeDistortionCfg,
     PhysicsMaterialCfg,
     PinholeCameraCfg,
     PreviewSurfaceCfg,

source/isaaclab/isaaclab/sim/spawners/__init__.pyi

@@ -54,6 +54,9 @@ __all__ = [
     "spawn_camera",
     "spawn_sensor_frame",
     "FisheyeCameraCfg",
+    "OpenCvDistortionCfg",
+    "OpenCvFisheyeDistortionCfg",
+    "OpenCvPinholeDistortionCfg",
     "PinholeCameraCfg",
     "SensorFrameCfg",
     "spawn_capsule",
@@ -126,7 +129,16 @@ from .meshes import (
     MeshRectangleCfg,
     MeshSphereCfg,
 )
-from .sensors import spawn_camera, spawn_sensor_frame, FisheyeCameraCfg, PinholeCameraCfg, SensorFrameCfg
+from .sensors import (
+    spawn_camera,
+    spawn_sensor_frame,
+    FisheyeCameraCfg,
+    OpenCvDistortionCfg,
+    OpenCvFisheyeDistortionCfg,
+    OpenCvPinholeDistortionCfg,
+    PinholeCameraCfg,
+    SensorFrameCfg,
+)
 from .shapes import (
     spawn_capsule,
     spawn_cone,

source/isaaclab/isaaclab/sim/spawners/sensors/__init__.pyi

@@ -7,9 +7,19 @@ __all__ = [
     "spawn_camera",
     "spawn_sensor_frame",
     "FisheyeCameraCfg",
+    "OpenCvDistortionCfg",
+    "OpenCvFisheyeDistortionCfg",
+    "OpenCvPinholeDistortionCfg",
     "PinholeCameraCfg",
     "SensorFrameCfg",
 ]
 
 from .sensors import spawn_camera, spawn_sensor_frame
-from .sensors_cfg import FisheyeCameraCfg, PinholeCameraCfg, SensorFrameCfg
+from .sensors_cfg import (
+    FisheyeCameraCfg,
+    OpenCvDistortionCfg,
+    OpenCvFisheyeDistortionCfg,
+    OpenCvPinholeDistortionCfg,
+    PinholeCameraCfg,
+    SensorFrameCfg,
+)

source/isaaclab/isaaclab/sim/spawners/sensors/sensors.py

@@ -8,7 +8,7 @@
 import logging
 from typing import TYPE_CHECKING
 
-from pxr import Sdf, Usd
+from pxr import Gf, Sdf, Usd
 
 from isaaclab.sim.utils import change_prim_property, clone, create_prim, get_current_stage
 from isaaclab.utils import to_camel_case
@@ -48,6 +48,65 @@
 """
 
 
+# OpenCV lens-distortion models authored as the ``omni:lensdistortion:*`` USD API. The RTX/OVRTX
+# renderer honors these attributes natively; they are read back into ``camera.data.intrinsic_matrices``
+# by :meth:`~isaaclab.sensors.camera.Camera._update_intrinsic_matrices`.
+_OPENCV_DISTORTION_API_SCHEMAS = {
+    "opencvPinhole": "OmniLensDistortionOpenCvPinholeAPI",
+    "opencvFisheye": "OmniLensDistortionOpenCvFisheyeAPI",
+}
+"""Maps an OpenCV distortion model discriminator to its applied USD API schema name."""
+
+_OPENCV_DISTORTION_COEFFS = {
+    "opencvPinhole": ("k1", "k2", "k3", "k4", "k5", "k6", "p1", "p2", "s1", "s2", "s3", "s4"),
+    "opencvFisheye": ("k1", "k2", "k3", "k4"),
+}
+"""Maps an OpenCV distortion model discriminator to its distortion-coefficient field names."""
+
+
+def _author_opencv_distortion(prim: Usd.Prim, cfg: sensors_cfg.OpenCvDistortionCfg) -> None:
+    """Author an OpenCV lens-distortion model on a camera prim as the ``omni:lensdistortion:*`` API.
+
+    The attributes are authored explicitly (not through the generic camelCase loop of
+    :func:`spawn_camera`) because their names are namespaced (e.g. ``omni:lensdistortion:opencvPinhole:k1``)
+    and cannot be produced by :func:`~isaaclab.utils.to_camel_case`. Applying the schema only edits prim
+    metadata, so it survives ``stage.ExportToString()`` and does not require the schema to be registered.
+
+    Args:
+        prim: The camera prim to author the distortion model on.
+        cfg: The OpenCV distortion configuration.
+
+    Raises:
+        ValueError: If the distortion ``model`` is not a supported OpenCV model.
+    """
+    if cfg.model not in _OPENCV_DISTORTION_API_SCHEMAS:
+        raise ValueError(
+            f"Unsupported OpenCV distortion model: '{cfg.model}'. Supported models are:"
+            f" {list(_OPENCV_DISTORTION_API_SCHEMAS)}."
+        )
+    prefix = f"omni:lensdistortion:{cfg.model}"
+
+    # apply the schema and set the model discriminator token
+    prim.AddAppliedSchema(_OPENCV_DISTORTION_API_SCHEMAS[cfg.model])
+
+    def _set_attr(name: str, type_name: Sdf.ValueTypeName, value) -> None:
+        attr = prim.GetAttribute(name) or prim.CreateAttribute(name, type_name)
+        attr.Set(value)
+
+    _set_attr("omni:lensdistortion:model", Sdf.ValueTypeNames.Token, cfg.model)
+    _set_attr(
+        f"{prefix}:imageSize",
+        Sdf.ValueTypeNames.Int2,
+        Gf.Vec2i(int(cfg.image_size[0]), int(cfg.image_size[1])),
+    )
+    for name in ("fx", "fy", "cx", "cy"):
+        _set_attr(f"{prefix}:{name}", Sdf.ValueTypeNames.Float, float(getattr(cfg, name)))
+    # coefficients are muted (authored as zero) unless apply_lens_distortion is set
+    for name in _OPENCV_DISTORTION_COEFFS[cfg.model]:
+        value = float(getattr(cfg, name)) if cfg.apply_lens_distortion else 0.0
+        _set_attr(f"{prefix}:{name}", Sdf.ValueTypeNames.Float, value)
+
+
 @clone
 def spawn_camera(
     prim_path: str,
@@ -119,6 +178,7 @@ def spawn_camera(
         "semantic_tags",
         "from_intrinsic_matrix",
         "spawn_path",
+        "distortion",
     ]
     # get camera prim
     prim = stage.GetPrimAtPath(prim_path)
@@ -143,6 +203,9 @@ def spawn_camera(
             prim_prop_name = to_camel_case(param_name, to="cC")
         # get attribute from the class
         prim.GetAttribute(prim_prop_name).Set(param_value)
+    # author the OpenCV lens-distortion model (renderer-agnostic; RTX/OVRTX honors it natively)
+    if getattr(cfg, "distortion", None) is not None:
+        _author_opencv_distortion(prim, cfg.distortion)
     # return the prim
     return prim