Manus + vive extension

build.sh

@@ -25,5 +25,10 @@ if [ $EULA_STATUS -ne 0 ]; then
     exit 1
 fi
 
+# Provision libsurvive into _build/target-deps on Linux (no-op if already installed)
+if [[ "$(uname -s)" == "Linux" ]]; then
+    bash "${SCRIPT_DIR}/source/scripts/build_libsurvive.sh" "${SCRIPT_DIR}"
+fi
+
 set -e
 source "$SCRIPT_DIR/repo.sh" build $@ || exit $?

source/apps/isaacsim.exp.base.kit

@@ -44,6 +44,7 @@ keywords = ["experience", "app", "usd"] # That makes it browsable in UI with "ex
 "isaacsim.simulation_app" = {}
 "isaacsim.storage.native" = {}
 "isaacsim.util.debug_draw" = {}
+"isaacsim.xr.input_devices" = {}
 "omni.isaac.core_archive" = {}
 "omni.isaac.ml_archive" = {}
 "omni.kit.loop-isaac" = {}

source/apps/isaacsim.exp.extscache.kit

@@ -101,6 +101,7 @@ folders = [
 "isaacsim.util.debug_draw" = {}
 "isaacsim.util.merge_mesh" = {}
 "isaacsim.util.physics" = {}
+"isaacsim.xr.input_devices" = {}
 "isaacsim.xr.openxr" = {}
 "omni.exporter.urdf" = {}
 "omni.isaac.app.selector" = {}

source/extensions/isaacsim.core.includes/include/isaacsim/core/includes/core/Platform.h

@@ -15,6 +15,7 @@
 
 #pragma once
 
+#include <cstdint>
 #include <string>
 #include <vector>
 

source/extensions/isaacsim.xr.input_devices/bindings/isaacsim.xr.input_devices/IsaacsimXrInputDevicesBindings.cpp

@@ -0,0 +1,36 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+#include <carb/BindingsPythonUtils.h>
+#include <isaacsim/xr/input_devices/ManusTracker.h>
+
+CARB_BINDINGS("isaacsim.xr.input_devices.python")
+
+namespace
+{
+PYBIND11_MODULE(_isaac_xr_input_devices, m)
+{
+    using namespace carb;
+    using namespace isaacsim::xr::input_devices;
+
+    auto carbModule = py::module::import("carb");
+
+    py::class_<IsaacSimManusTracker>(m, "IsaacSimManusTracker")
+        .def(py::init<>())
+        .def("initialize", &IsaacSimManusTracker::initialize, R"(
+            Initialize Manus SDK (adapted from existing implementation).
+
+            Returns:
+                bool: True if initialization was successful, False otherwise
+        )")
+        .def("get_glove_data", &IsaacSimManusTracker::get_glove_data, R"(
+            Get glove data in IsaacSim format.
+
+            Returns:
+                Dict[str, List[float]]: Dictionary mapping glove data keys to values
+        )")
+        .def("cleanup", &IsaacSimManusTracker::cleanup, R"(
+            Cleanup SDK resources.
+        )");
+}
+}

source/extensions/isaacsim.xr.input_devices/config/extension.toml

@@ -0,0 +1,31 @@
+[package]
+version = "1.0.0"
+category = "XR"
+title = "Isaac Sim XR Input Devices"
+description = "Provides XR input device support for Manus gloves and Vive trackers"
+authors = ["[email protected]"]
+repository = "https://gitlab-master.nvidia.com/omniverse/isaac/omni_isaac_sim/-/tree/develop/source/extensions/isaacsim.xr.input_devices"
+keywords = ["isaac", "xr", "manus", "vive", "trackers", "gloves", "input", "devices"]
+changelog = "docs/CHANGELOG.md"
+readme = "docs/README.md"
+writeTarget.kit = true
+
+[dependencies]
+"omni.kit.xr.core" = {}
+"isaacsim.core.api" = {}
+"isaacsim.core.deprecation_manager" = {}
+"isaacsim.core.utils" = {}
+
+[[python.module]]
+name = "isaacsim.xr.input_devices"
+
+[[test]]
+dependencies = [
+    "isaacsim.core.api",
+    "isaacsim.core.utils",
+]
+args = [
+    "--/app/asyncRendering=0",
+    "--/app/fastShutdown=1",
+    "--vulkan",
+]

source/extensions/isaacsim.xr.input_devices/docs/CHANGELOG.md

@@ -0,0 +1,35 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [1.0.0] - 2025-08-08
+
+### Added
+- Initial release of `isaacsim.xr.input_devices` extension
+- Manus gloves integration via C++ tracker with Python bindings
+- Vive tracker integration via `pysurvive` (libsurvive) with mock fallback
+- Unified Python API: `get_xr_device_integration().get_all_device_data()`
+- Left/right wrist mapping resolution between Vive trackers and OpenXR wrists
+- Static scene-to-lighthouse transform estimation with clustering and averaging
+- Per-device connection status and last-data timestamps
+- Sample visualization script for Manus and Vive devices
+- Documentation and simple CLI-style tests
+
+### Details
+- Update cadence is rate-limited to 100 Hz
+- Wrist mapping considers both pairings (WM0→L/WM1→R vs WM1→L/WM0→R),
+  accumulates per-frame translation/rotation errors, and chooses the better pairing
+- Chosen pairing candidates are clustered (position/orientation margins) and averaged
+  to produce a stable `scene_T_lighthouse_static` transform
+- Vive tracker poses are transformed into scene coordinates using the static transform
+- Manus joint poses (relative to the wrist) and transformed into scene coordinates 
+  using the resolved wrist mapping from vive tracker poses
+
+### Dependencies
+- Isaac Sim Core APIs
+- Omniverse Kit runtime (for logging and extension lifecycle)
+- Manus SDK (optional; otherwise uses mock data)
+- libsurvive / pysurvive (optional; otherwise uses mock data)

source/extensions/isaacsim.xr.input_devices/docs/README.md

@@ -0,0 +1,163 @@
+# IsaacSim XR Input Devices Extension
+
+This extension provides XR input device support for Manus gloves and Vive trackers, with a unified Python API and automatic wrist mapping between trackers and OpenXR wrists.
+
+## Features
+
+- **Manus gloves**: Real-time hand/finger joint poses via C++ tracker with Python bindings
+- **Vive trackers**: 6DOF poses via libsurvive (`pysurvive`) with mock fallback
+- **Left/Right mapping**: Resolves which Vive tracker corresponds to left/right wrist
+- **Scene alignment**: Estimates a stable scene↔lighthouse transform from early samples
+- **Unified API**: Single call to fetch all device data and device status
+- **Visualization sample**: Renders gloves/trackers as cubes in Isaac Sim
+
+## Prerequisites
+
+### Hardware
+- **Manus Gloves**: Manus Prime/MetaGloves with license dongle
+- **Vive Trackers**: Lighthouse tracking (SteamVR base stations + trackers)
+
+### Software
+- **Manus SDK**: Bundled in Isaac Sim target-deps (mock used if missing)
+- **libsurvive / pysurvive**: Required for real Vive tracking; otherwise mock data is used
+
+Install libsurvive (either system-wide or in your home directory):
+
+System-wide:
+```bash
+sudo apt update
+sudo apt install -y build-essential zlib1g-dev libx11-dev libusb-1.0-0-dev \
+  freeglut3-dev liblapacke-dev libopenblas-dev libatlas-base-dev cmake
+
+git clone https://github.com/cntools/libsurvive.git
+cd libsurvive
+sudo cp ./useful_files/81-vive.rules /etc/udev/rules.d/
+sudo udevadm control --reload-rules && sudo udevadm trigger
+make && cmake . && make -j"$(nproc)"
+sudo make install && sudo ldconfig
+```
+
+User (home) install (recommended during development):
+```bash
+git clone https://github.com/cntools/libsurvive.git ~/libsurvive
+cd ~/libsurvive
+sudo cp ./useful_files/81-vive.rules /etc/udev/rules.d/
+sudo udevadm control --reload-rules && sudo udevadm trigger
+make && cmake . && make -j"$(nproc)"
+
+# Ensure Python can find pysurvive bindings
+export PYTHONPATH="$HOME/libsurvive/bindings/python:$PYTHONPATH"
+```
+
+Note: The Vive tracker wrapper currently prepends a libsurvive Python path. Adjust it as needed for your environment or set `PYTHONPATH` as shown above.
+
+## Building
+
+```bash
+# Build Isaac Sim (includes this extension)
+./build.sh
+```
+
+## Usage
+
+### Sample
+Run the visualization sample that renders Manus joints (red cubes) and Vive trackers (blue cubes):
+
+```bash
+cd /path/to/IsaacSim
+./_build/linux-x86_64/release/python.sh \
+  source/standalone_examples/api/isaacsim.xr.input_devices/manus_vive_tracking_sample.py
+```
+
+### Python API
+
+```python
+from isaacsim.xr.input_devices.impl.xr_device_integration import get_xr_device_integration
+
+# Obtain the shared integration instance from the extension
+integration = get_xr_device_integration()
+
+# Fetch all device data
+all_data = integration.get_all_device_data()
+
+manus_data = all_data.get('manus_gloves', {})
+vive_data = all_data.get('vive_trackers', {})
+status = all_data.get('device_status', {})
+
+print(f"Manus connected: {status.get('manus_gloves', {}).get('connected', False)}")
+print(f"Vive connected: {status.get('vive_trackers', {}).get('connected', False)}")
+```
+
+### Data Format
+
+```python
+{
+  'manus_gloves': {
+    'left_0': {
+      'position': [x, y, z],
+      'orientation': [w, x, y, z]
+    },
+    'left_1': { ... },
+    'right_0': { ... },
+    # ... per-joint entries
+  },
+  'vive_trackers': {
+    '<device_id>': {
+      'position': [x, y, z],
+      'orientation': [w, x, y, z]
+    },
+    # e.g., 'WM0', 'WM1', or device names from libsurvive
+  },
+  'device_status': {
+    'manus_gloves': {'connected': bool, 'last_data_time': float},
+    'vive_trackers': {'connected': bool, 'last_data_time': float},
+    'left_hand_connected': bool,
+    'right_hand_connected': bool
+  }
+}
+```
+
+## How left/right mapping is determined
+
+- Detect connected OpenXR wrists (left/right) and available Vive wrist markers (e.g., WM0/WM1)
+- For each frame, compute candidate transforms for both pairings:
+  - Pair A: WM0→Left, WM1→Right
+  - Pair B: WM1→Left, WM0→Right
+- Accumulate translation/rotation deltas per pairing when both wrists and trackers are present
+- Choose the pairing:
+  - Prefer the pairing with more samples initially
+  - Once there are enough paired frames, choose the one with lower accumulated error
+- Cluster the chosen pairing’s transforms and average them to estimate a stable
+  scene↔lighthouse transform (`scene_T_lighthouse_static`)
+- Use the resolved mapping and transform to place all Vive and Manus data in scene coordinates
+
+## Troubleshooting
+
+- Manus license issues: replug license dongle; ensure SDK libraries are discoverable
+- libsurvive conflicts: ensure SteamVR is NOT running concurrently
+- No Vive devices: verify udev rules and USB connections (solid tracker LED)
+- Python import for `pysurvive`: set `PYTHONPATH` to libsurvive bindings path
+
+## Extension Layout
+
+```
+isaacsim.xr.input_devices/
+├── bindings/                    # Pybind11 bindings
+├── include/                     # C++ headers
+├── plugins/                     # C++ implementations
+├── python/
+│   └── impl/
+│       ├── extension.py                 # Extension lifecycle
+│       ├── xr_device_integration.py     # Orchestrates devices & transforms
+│       ├── manus_tracker.py             # Manus wrapper
+│       └── vive_tracker.py              # Vive wrapper
+└── docs/
+    ├── README.md
+    └── CHANGELOG.md
+```
+
+## License
+
+Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+
+SPDX-License-Identifier: Apache-2.0 

source/extensions/isaacsim.xr.input_devices/include/isaacsim/xr/input_devices/ManusTracker.h

@@ -0,0 +1,71 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+#pragma once
+
+#include <memory>
+#include <mutex>
+#include <optional>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "ManusSDK.h"
+
+#ifdef _MSC_VER
+#    if ISAACSIM_XR_INPUT_DEVICES_EXPORT
+#        define ISAACSIM_XR_INPUT_DEVICES_DLL_EXPORT __declspec(dllexport)
+#    else
+#        define ISAACSIM_XR_INPUT_DEVICES_DLL_EXPORT __declspec(dllimport)
+#    endif
+#else
+#    define ISAACSIM_XR_INPUT_DEVICES_DLL_EXPORT __attribute__((visibility("default")))
+#endif
+
+namespace isaacsim
+{
+namespace xr
+{
+namespace input_devices
+{
+
+class ISAACSIM_XR_INPUT_DEVICES_DLL_EXPORT IsaacSimManusTracker
+{
+public:
+    IsaacSimManusTracker();
+    ~IsaacSimManusTracker();
+
+    bool initialize();
+    std::unordered_map<std::string, std::vector<float>> get_glove_data();
+    void cleanup();
+
+private:
+    static IsaacSimManusTracker* s_instance;
+    static std::mutex s_instance_mutex;
+
+    // ManusSDK specific members
+    void RegisterCallbacks();
+    void ConnectToGloves();
+    void DisconnectFromGloves();
+
+    // Callback functions
+    static void OnSkeletonStream(const SkeletonStreamInfo* skeleton_stream_info);
+    static void OnLandscapeStream(const Landscape* landscape);
+    static void OnErgonomicsStream(const ErgonomicsStream* ergonomics_stream);
+
+    // Data storage (following isaac-deploy pattern)
+    std::mutex output_map_mutex;
+    std::mutex landscape_mutex;
+    std::unordered_map<std::string, std::vector<float>> output_map;
+    std::optional<uint32_t> left_glove_id;
+    std::optional<uint32_t> right_glove_id;
+    bool is_connected = false;
+
+    // Legacy member for compatibility
+    std::unordered_map<std::string, std::vector<float>> m_glove_data;
+    std::mutex m_data_mutex;
+};
+
+} // namespace input_devices
+} // namespace xr
+} // namespace isaacsim