update

examples/hitl/rearrange_v2/config/hitl_single_test.yaml

@@ -3,19 +3,23 @@
 # Top-level configuration for running a headless HITL server in single-user mode.
 
 defaults:
-  - lang_rearrange_humanoid_only.yaml
+  - lang_rearrange_spot_only.yaml
   - _self_
 
 
 rearrange_v2:
   highlight_default_receptacles: True
   agents:
-    agent_0:
+    main_agent:
       head_sensor_substring: "head"
       can_change_object_states: True
 
 SHORT_CIRCUIT: True
 habitat_hitl:
+  window: ~
   networking:
     enable: True
-    max_client_count: 1
+    max_client_count: 1
+  experimental:
+    headless:
+      do_headless: True

examples/hitl/rearrange_v2/config/lang_rearrange_spot_only.yaml

@@ -0,0 +1,36 @@
+# @package _global_
+
+defaults:
+  - language_rearrange_spot
+  - hitl_defaults
+  - dataset
+  - _self_
+
+habitat_baselines:
+  # todo: document these choices
+  eval:
+    should_load_ckpt: False
+  rl:
+    agent:
+      num_pool_agents_per_type: [1, 1]
+    policy:
+
+habitat_hitl:
+  window:
+    title: "Rearrange"
+    width: 1300
+    height: 1000
+  gui_controlled_agents:
+    - agent_index: 0
+      lin_speed: 10.0
+      ang_speed: 300
+  hide_humanoid_in_gui: False
+  can_grasp_place_threshold: 2.0
+  camera:
+    first_person_mode: True
+  networking:
+    client_sync:
+      server_camera: False
+      server_input: False
+    client_max_idle_duration: 180.0
+    enable_connections_by_default: False

examples/hitl/rearrange_v2/config/language_rearrange_spot.yaml

@@ -0,0 +1,75 @@
+# This config is derived from habitat-lab/habitat/config/benchmark/multi_agent/hssd_spot_human.yaml
+# @package _global_
+
+defaults:
+  - /habitat: habitat_config_base
+  - /habitat/task: task_config_base
+
+  - /habitat/simulator/[email protected]_agent: rgbd_head_agent
+  - /habitat/simulator/[email protected]_agent: spot
+
+  - /habitat/dataset/rearrangement: hssd
+
+  - /habitat/task/[email protected]_velocity: base_velocity
+  - /habitat/task/[email protected]_stop: rearrange_stop
+  - /habitat/task/[email protected]_action: arm_action
+
+  - /habitat/task/measurements:
+    - num_steps
+  - /habitat/task/lab_sensors:
+    - relative_resting_pos_sensor
+    - target_start_sensor
+    - goal_sensor
+    - joint_sensor
+    - is_holding_sensor
+    - end_effector_sensor
+    - target_start_gps_compass_sensor
+    - target_goal_gps_compass_sensor
+    - localization_sensor
+
+  - _self_
+
+habitat:
+  task:
+    type: RearrangeEmptyTask-v0
+    reward_measure: num_steps
+    success_measure: num_steps
+    success_reward: 10.0
+    min_distance_start_agents: 5.0
+    slack_reward: -0.0005
+    end_on_success: True
+    constraint_violation_ends_episode: False
+    constraint_violation_drops_object: False
+    task_spec_base_path: benchmark/multi_agent/
+    task_spec: pddl/multi_agent_tidy_house
+    pddl_domain_def: fp
+    actions:
+      base_velocity:
+        lin_speed: 10.0
+        ang_speed: 300
+
+    robot_at_thresh: 3.0
+  gym:
+    obs_keys:
+      - is_holding
+      - ee_pos
+      - localization_sensor
+  simulator:
+    type: CollaborationSim-v0
+    seed: 100
+    concur_render: True
+    auto_sleep: True
+    agents_order:
+      - main_agent
+
+    agents:
+      main_agent:
+        auto_update_sensor_transform: False
+
+    kinematic_mode: True
+    ac_freq_ratio: 1
+    step_physics: False
+
+    habitat_sim_v0:
+      allow_sliding: True
+      enable_physics: True

habitat-hitl/habitat_hitl/environment/controllers/controller_helper.py

@@ -96,6 +96,7 @@ def __init__(
                     )
 
                     gui_agent_controller: Controller
+
                     if articulated_agent_type == "KinematicHumanoid":
                         gui_agent_controller = GuiHumanoidController(
                             agent_idx=agent_index,
@@ -110,7 +111,10 @@ def __init__(
                             ),
                         )
                     elif articulated_agent_type == "SpotRobot":
-                        agent_k = f"agent_{agent_index}"
+                        if is_multi_agent:
+                            agent_k = f"agent_{agent_index}_"
+                        else:
+                            agent_k = ""
                         original_action_space = clean_dict(
                             self._gym_habitat_env.original_action_space,
                             agent_k,
@@ -123,7 +127,7 @@ def __init__(
                             base_vel_action_idx,
                             base_vel_action_end_idx,
                         ) = find_action_range(
-                            original_action_space, "_base_velocity"
+                            original_action_space, "base_velocity"
                         )
 
                         assert len(action_space.shape) == 1
@@ -135,7 +139,7 @@ def __init__(
                         turn_scale = (
                             config.habitat.simulator.ctrl_freq
                             / config.habitat.task.actions[
-                                f"{agent_k}_base_velocity"
+                                f"{agent_k}base_velocity"
                             ].ang_speed
                         )
 
@@ -216,8 +220,8 @@ def update(self, obs):
             raise ValueError(
                 "ControllerHelper only supports up to 2 controllers."
             )
-
-        return action
+        # convert action to float32
+        return np.float32(action)
 
     def on_environment_reset(self):
         for controller in self.controllers:

habitat-hitl/habitat_hitl/environment/controllers/gui_controller.py

@@ -111,7 +111,6 @@ def act(self, obs, env):
         base_name = f"{agent_k}base_velocity"
         ac_spaces = env.action_space.spaces
 
-        assert base_name in ac_spaces
         base_action_space = ac_spaces[base_name][base_k]
         base_action = np.zeros(base_action_space.shape[0])