From 6b0c77842070b9dd1987403977dfd9220f8e4785 Mon Sep 17 00:00:00 2001 From: Xavier Puig Date: Tue, 22 Apr 2025 01:03:14 +0000 Subject: [PATCH] update --- .../rearrange_v2/config/hitl_single_test.yaml | 10 ++- .../config/lang_rearrange_spot_only.yaml | 36 +++++++++ .../config/language_rearrange_spot.yaml | 75 +++++++++++++++++++ .../controllers/controller_helper.py | 14 ++-- .../environment/controllers/gui_controller.py | 1 - 5 files changed, 127 insertions(+), 9 deletions(-) create mode 100644 examples/hitl/rearrange_v2/config/lang_rearrange_spot_only.yaml create mode 100644 examples/hitl/rearrange_v2/config/language_rearrange_spot.yaml diff --git a/examples/hitl/rearrange_v2/config/hitl_single_test.yaml b/examples/hitl/rearrange_v2/config/hitl_single_test.yaml index 25b17ccd40..a5e18283d7 100644 --- a/examples/hitl/rearrange_v2/config/hitl_single_test.yaml +++ b/examples/hitl/rearrange_v2/config/hitl_single_test.yaml @@ -3,19 +3,23 @@ # Top-level configuration for running a headless HITL server in single-user mode. defaults: - - lang_rearrange_humanoid_only.yaml + - lang_rearrange_spot_only.yaml - _self_ rearrange_v2: highlight_default_receptacles: True agents: - agent_0: + main_agent: head_sensor_substring: "head" can_change_object_states: True SHORT_CIRCUIT: True habitat_hitl: + window: ~ networking: enable: True - max_client_count: 1 \ No newline at end of file + max_client_count: 1 + experimental: + headless: + do_headless: True diff --git a/examples/hitl/rearrange_v2/config/lang_rearrange_spot_only.yaml b/examples/hitl/rearrange_v2/config/lang_rearrange_spot_only.yaml new file mode 100644 index 0000000000..48e61dc5d0 --- /dev/null +++ b/examples/hitl/rearrange_v2/config/lang_rearrange_spot_only.yaml @@ -0,0 +1,36 @@ +# @package _global_ + +defaults: + - language_rearrange_spot + - hitl_defaults + - dataset + - _self_ + +habitat_baselines: + # todo: document these choices + eval: + should_load_ckpt: False + rl: + agent: + num_pool_agents_per_type: [1, 1] + policy: + +habitat_hitl: + window: + title: "Rearrange" + width: 1300 + height: 1000 + gui_controlled_agents: + - agent_index: 0 + lin_speed: 10.0 + ang_speed: 300 + hide_humanoid_in_gui: False + can_grasp_place_threshold: 2.0 + camera: + first_person_mode: True + networking: + client_sync: + server_camera: False + server_input: False + client_max_idle_duration: 180.0 + enable_connections_by_default: False diff --git a/examples/hitl/rearrange_v2/config/language_rearrange_spot.yaml b/examples/hitl/rearrange_v2/config/language_rearrange_spot.yaml new file mode 100644 index 0000000000..452a44683b --- /dev/null +++ b/examples/hitl/rearrange_v2/config/language_rearrange_spot.yaml @@ -0,0 +1,75 @@ +# This config is derived from habitat-lab/habitat/config/benchmark/multi_agent/hssd_spot_human.yaml +# @package _global_ + +defaults: + - /habitat: habitat_config_base + - /habitat/task: task_config_base + + - /habitat/simulator/sensor_setups@habitat.simulator.agents.main_agent: rgbd_head_agent + - /habitat/simulator/agents@habitat.simulator.agents.main_agent: spot + + - /habitat/dataset/rearrangement: hssd + + - /habitat/task/actions@habitat.task.actions.base_velocity: base_velocity + - /habitat/task/actions@habitat.task.actions.rearrange_stop: rearrange_stop + - /habitat/task/actions@habitat.task.actions.arm_action: arm_action + + - /habitat/task/measurements: + - num_steps + - /habitat/task/lab_sensors: + - relative_resting_pos_sensor + - target_start_sensor + - goal_sensor + - joint_sensor + - is_holding_sensor + - end_effector_sensor + - target_start_gps_compass_sensor + - target_goal_gps_compass_sensor + - localization_sensor + + - _self_ + +habitat: + task: + type: RearrangeEmptyTask-v0 + reward_measure: num_steps + success_measure: num_steps + success_reward: 10.0 + min_distance_start_agents: 5.0 + slack_reward: -0.0005 + end_on_success: True + constraint_violation_ends_episode: False + constraint_violation_drops_object: False + task_spec_base_path: benchmark/multi_agent/ + task_spec: pddl/multi_agent_tidy_house + pddl_domain_def: fp + actions: + base_velocity: + lin_speed: 10.0 + ang_speed: 300 + + robot_at_thresh: 3.0 + gym: + obs_keys: + - is_holding + - ee_pos + - localization_sensor + simulator: + type: CollaborationSim-v0 + seed: 100 + concur_render: True + auto_sleep: True + agents_order: + - main_agent + + agents: + main_agent: + auto_update_sensor_transform: False + + kinematic_mode: True + ac_freq_ratio: 1 + step_physics: False + + habitat_sim_v0: + allow_sliding: True + enable_physics: True diff --git a/habitat-hitl/habitat_hitl/environment/controllers/controller_helper.py b/habitat-hitl/habitat_hitl/environment/controllers/controller_helper.py index b96b206c47..d766933b1f 100644 --- a/habitat-hitl/habitat_hitl/environment/controllers/controller_helper.py +++ b/habitat-hitl/habitat_hitl/environment/controllers/controller_helper.py @@ -96,6 +96,7 @@ def __init__( ) gui_agent_controller: Controller + if articulated_agent_type == "KinematicHumanoid": gui_agent_controller = GuiHumanoidController( agent_idx=agent_index, @@ -110,7 +111,10 @@ def __init__( ), ) elif articulated_agent_type == "SpotRobot": - agent_k = f"agent_{agent_index}" + if is_multi_agent: + agent_k = f"agent_{agent_index}_" + else: + agent_k = "" original_action_space = clean_dict( self._gym_habitat_env.original_action_space, agent_k, @@ -123,7 +127,7 @@ def __init__( base_vel_action_idx, base_vel_action_end_idx, ) = find_action_range( - original_action_space, "_base_velocity" + original_action_space, "base_velocity" ) assert len(action_space.shape) == 1 @@ -135,7 +139,7 @@ def __init__( turn_scale = ( config.habitat.simulator.ctrl_freq / config.habitat.task.actions[ - f"{agent_k}_base_velocity" + f"{agent_k}base_velocity" ].ang_speed ) @@ -216,8 +220,8 @@ def update(self, obs): raise ValueError( "ControllerHelper only supports up to 2 controllers." ) - - return action + # convert action to float32 + return np.float32(action) def on_environment_reset(self): for controller in self.controllers: diff --git a/habitat-hitl/habitat_hitl/environment/controllers/gui_controller.py b/habitat-hitl/habitat_hitl/environment/controllers/gui_controller.py index d13dd728b9..8f157db82f 100644 --- a/habitat-hitl/habitat_hitl/environment/controllers/gui_controller.py +++ b/habitat-hitl/habitat_hitl/environment/controllers/gui_controller.py @@ -111,7 +111,6 @@ def act(self, obs, env): base_name = f"{agent_k}base_velocity" ac_spaces = env.action_space.spaces - assert base_name in ac_spaces base_action_space = ac_spaces[base_name][base_k] base_action = np.zeros(base_action_space.shape[0])