Skip to content

Fix example demos: tensor handling, keyboard teleop #1952

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
kmy17518 wants to merge 7 commits into
base: main
Choose a base branch
from
+42 −19
Open

Fix example demos: tensor handling, keyboard teleop #1952

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
27f8212
Constrain dependencies for JoyLo installation for compatibility with …
kmy17518 Jan 23, 2026
10620c8
Fix tensor handling and keyboard teleop
kmy17518 Jan 23, 2026
ac12a6f
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Jan 23, 2026
476d8de
Fix crash when recording trajectory with too few waypoints
kmy17518 Jan 23, 2026
36ad54a
Fix key release handling in KeyboardRobotController to track specific…
kmy17518 Jan 23, 2026
f5b0eae
Use mktemp for JoyLo constraints file
kmy17518 Jan 23, 2026
e3fd762
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Jan 23, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 15 additions & 7 deletions OmniGibson/omnigibson/examples/scenes/scene_tour_demo.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,20 @@ def clear_waypoints():
print("Cleared all waypoints!")
waypoints = []

def record_trajectory():
if len(waypoints) < 3:
print(
f"Need at least 3 waypoints to record a trajectory (currently have {len(waypoints)}). Press X to add waypoints."
)
return
cam_mover.record_trajectory_from_waypoints(
waypoints=th.stack(waypoints),
per_step_distance=0.02,
fps=30,
steps_per_frame=1,
fpath=None, # This corresponds to the default path inferred from cam_mover.save_dir
)

KeyboardEventHandler.initialize()
KeyboardEventHandler.add_keyboard_callback(
key=lazy.carb.input.KeyboardInput.X,
Expand All @@ -74,13 +88,7 @@ def clear_waypoints():
)
KeyboardEventHandler.add_keyboard_callback(
key=lazy.carb.input.KeyboardInput.J,
callback_fn=lambda: cam_mover.record_trajectory_from_waypoints(
waypoints=th.tensor(waypoints),
per_step_distance=0.02,
fps=30,
steps_per_frame=1,
fpath=None, # This corresponds to the default path inferred from cam_mover.save_dir
),
callback_fn=record_trajectory,
)
KeyboardEventHandler.add_keyboard_callback(
key=lazy.carb.input.KeyboardInput.ESCAPE,
Expand Down
28 changes: 17 additions & 11 deletions OmniGibson/omnigibson/utils/ui_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -405,7 +405,7 @@ def record_image(self, fpath=None):

# Make sure save path directory exists, and then save the image to that location
Path(Path(fpath).parent).mkdir(parents=True, exist_ok=True)
Image.fromarray(self.get_image()).save(fpath)
Image.fromarray(self.get_image().cpu().numpy()).save(fpath)
og.log.info(f"Saved current viewer camera image to {fpath}.")

def record_trajectory(self, poses, fps, steps_per_frame=1, fpath=None):
Expand Down Expand Up @@ -438,7 +438,7 @@ def record_trajectory(self, poses, fps, steps_per_frame=1, fpath=None):
self.cam.set_position_orientation(position=pos, orientation=quat)
og.sim.step()
if i % steps_per_frame == 0:
video_writer.append_data(self.get_image())
video_writer.append_data(self.get_image().cpu().numpy())

# Close writer
video_writer.close()
Expand Down Expand Up @@ -470,7 +470,7 @@ def record_trajectory_from_waypoints(self, waypoints, per_step_distance, fps, st

# Function help get arc derivative
def arc_derivative(u):
return th.sqrt(th.sum([dspline(u) ** 2 for dspline in dsplines]))
return math.sqrt(sum(dspline(u) ** 2 for dspline in dsplines))

# Function to help get interpolated positions
def get_interpolated_positions(step):
Expand All @@ -480,7 +480,7 @@ def get_interpolated_positions(step):
interpolated_points = th.zeros((path_length, 3))
for i in range(path_length):
curr_step = step + (i / path_length)
interpolated_points[i, :] = th.tensor([spline(curr_step) for spline in splines])
interpolated_points[i, :] = th.tensor([float(spline(curr_step)) for spline in splines])
return interpolated_points

# Iterate over all waypoints and infer the resulting trajectory, recording the resulting poses
Expand All @@ -497,7 +497,7 @@ def get_interpolated_positions(step):
pan_angle = th.arctan2(-xy_direction[0], xy_direction[1])
tilt_angle = th.arcsin(z)
# Infer global quat orientation from these angles
quat = T.euler2quat([math.pi / 2 + tilt_angle, 0.0, pan_angle])
quat = T.euler2quat(th.tensor([math.pi / 2 + tilt_angle.item(), 0.0, pan_angle.item()]))
poses.append([positions[j], quat])

# Record the generated trajectory
Expand Down Expand Up @@ -624,6 +624,7 @@ def __init__(self, robot):
self.keypress_mapping = None # Maps omni keybindings to information for controlling various parts of the robot
self.current_keypress = None # Current key that is being pressed
self.active_action = None # Current action information based on the current keypress
self._pending_release_key = None # Which key has a pending release (to handle batched events)
self.toggling_gripper = False # Whether we should toggle the gripper during the next action
self.custom_keymapping = None # Dictionary mapping custom keys to custom callback functions / info

Expand All @@ -640,7 +641,7 @@ def register_keyboard_handler(self):
appwindow = lazy.omni.appwindow.get_default_app_window()
input_interface = lazy.carb.input.acquire_input_interface()
keyboard = appwindow.get_keyboard()
input_interface.subscribe_to_keyboard_events(keyboard, self.keyboard_event_handler)
self._keyboard_sub = input_interface.subscribe_to_keyboard_events(keyboard, self.keyboard_event_handler)

def register_custom_keymapping(self, key, description, callback_fn):
"""
Expand Down Expand Up @@ -837,10 +838,9 @@ def keyboard_event_handler(self, event, *args, **kwargs):
if event.input == lazy.carb.input.KeyboardInput.T:
self.toggling_gripper = True

# If we release a key, clear the active action and keypress
# If we release a key, mark release as pending (don't clear immediately to handle batched events)
elif event.type == lazy.carb.input.KeyboardEventType.KEY_RELEASE:
self.active_action = None
self.current_keypress = None
self._pending_release_key = event.input

# Callback always needs to return True
return True
Expand Down Expand Up @@ -917,6 +917,12 @@ def get_teleop_action(self):
# print("Pressed {}. Action: {}".format(keypress_str, action.tolist()))
# sys.stdout.write("\033[F")

# Clear action after consuming if a key release was pending for the current key
if self._pending_release_key is not None and self._pending_release_key == self.current_keypress:
self.active_action = None
self.current_keypress = None
self._pending_release_key = None

# Return action
return action

Expand All @@ -939,8 +945,8 @@ def print_command(char, info):
print_command("[, ]", "move the joint backwards, forwards, respectively")
print()
print("Differential Drive Control")
print_command("i, k", "turn left, right")
print_command("l, j", "move forward, backwards")
print_command("j, l", "turn left, right")
print_command("i, k", "move forward, backwards")
print()
print("Inverse Kinematics Control")
print_command("3, 4", "toggle between the different arm(s) to control")
Expand Down
11 changes: 10 additions & 1 deletion setup.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -371,7 +371,16 @@ fi
if [ "$JOYLO" = true ]; then
echo "Installing JoyLo..."
[ ! -d "joylo" ] && { echo "ERROR: joylo directory not found"; exit 1; }
pip install -e "$WORKDIR/joylo"
# Constrain deps to keep OmniGibson / Isaac Sim compatible
JOYLO_CONSTRAINTS_FILE=$(mktemp)
trap 'rm -f "$JOYLO_CONSTRAINTS_FILE"' EXIT
cat > "$JOYLO_CONSTRAINTS_FILE" << 'EOF'
opencv-contrib-python<=4.11.0.86
numpy<2
EOF
pip install -e "$WORKDIR/joylo" -c "$JOYLO_CONSTRAINTS_FILE"
rm -f "$JOYLO_CONSTRAINTS_FILE"
trap - EXIT
fi

# Install Eval
Expand Down