Twix mujoco visualizer (#2143)

* Add kinematics to sensor_data_receiver

* Add bevy rendering to twix

* Parse mujoco scene in twix

* Use egui fork

* Make meshes not fuchsia anymore

* Compute normals to get proper lighting

* Its alive!!!!

* Request redraw when an update arrives

* Retry connection until it succeeds

* Fix axes

* Extract processing scene updates from ui function

* Implement resizing of the viewport

* Use physical pixels instead of logical ui coordinates to calculate texture size

* Forward necessary egui input to bevy to get pan-orbit-camera working

* Rotate the scene instead of placing the camera weirdly to fix the pan-orbit-camera

* Disable unimplemented events for now to prevent todo!()-crashes

* Fix lints

* Don't rotate everything all the time

* Turn the robot right-side-up

* Ignore input which does not belong to the bevy view

* Request and receive initial scene description via websocket

* Don't load /tmp/scene at startup

* Forward mouse wheel events from egui to bevy to allow zooming the camera

* Use github fork of egui for patching crates.io

* Fix clippy lints: twix

* Port bevyhavior_simulator to bevy 0.16

* Draw gizmo at left wrist joint

* Draw remaining joints

* Attach trunk frame to the trunk body instead of the trunk mesh

* Fix toml issues

* Remove unused code for simulator control

* Use logging

* Yeet js visualizer

* Shorten websocket URL

* Deduplicate between camera creation and camera update

* Replace unwrap with expect

* Replace Vec of known size with array

* Check body name for "Trunk" instead of checking each mesh name

* Address Ole review comments

Author: knoellle

Cargo.lock

@@ -89,12 +89,12 @@ bat = { version = "0.25.0", default-features = false, features = [
   "paging",
   "regex-fancy",
 ] }
-bevy = { version = "0.15.2", default-features = false }
+bevy = { version = "0.16.0", default-features = false }
 bevy-inspector-egui = "0.29.1"
 bevy_asset_loader = "0.22.0"
 bevy_egui = "0.33.0"
 bevy_obj = "0.15"
-bevy_panorbit_camera = { version = "0.23.0", features = ["bevy_egui"] }
+bevy_panorbit_camera = { version = "0.26.0", features = ["bevy_egui"] }
 bincode = "1.3.3"
 bindgen = "0.71.1"
 blake3 = { version = "1.8.2", features = ["mmap", "serde"] }
@@ -122,10 +122,10 @@ derive_more = { version = "2.0.1", features = [
   "not",
 ] }
 dirs = "6.0.0"
-eframe = { version = "0.31.0", features = ["persistence"] }
-egui = { version = "0.31.0", features = ["persistence"] }
+eframe = { version = "0.31.1", features = ["persistence", "wgpu"] }
+egui = { version = "=0.31.1", features = ["persistence"] }
 egui_dock = { version = "0.16.0", features = ["serde"] }
-egui_extras = { version = "0.31.0", features = ["image"] }
+egui_extras = { version = "0.31.1", features = ["image"] }
 egui_plot = "0.31.0"
 energy_optimization = { path = "crates/energy_optimization" }
 enum-iterator = "2.1.0"
@@ -160,7 +160,11 @@ log = "0.4.25"
 mcap = "0.15.0"
 mlua = { version = "0.10.3", features = ["luajit", "serialize", "vendored"] }
 motionfile = { path = "crates/motionfile" }
-nalgebra = { version = "0.33.2", features = ["serde", "serde-serialize"] }
+nalgebra = { version = "0.33.2", features = [
+  "convert-glam029",
+  "serde",
+  "serde-serialize",
+] }
 nao = { path = "crates/nao" }
 ndarray = { version = "0.16.1", features = ["serde"] }
 ndarray-conv = "0.4.1"
@@ -261,6 +265,9 @@ serde_derive = { git = "https://github.com/HULKs/serde.git", rev = "ef30eb8fdd0e
 # Pinned to forked serde version since https://github.com/serde-rs/serde/pull/2513 is not merged
 serde_core = { git = "https://github.com/HULKs/serde.git", rev = "ef30eb8fdd0ee12e4e6f3f5615c6bb022e02d1b2" }
 
+egui = { git = "https://github.com/HULKs/egui/", rev = "7a3a88dcbb46a24d49a2388b2d9c8a6a273a9d4e" }
+egui-wgpu = { git = "https://github.com/HULKs/egui/", rev = "7a3a88dcbb46a24d49a2388b2d9c8a6a273a9d4e" }
+
 [profile.dev]
 opt-level = 3
 debug = false

Cargo.toml

@@ -4,8 +4,9 @@ use bevy::{
     app::{App, Update},
     ecs::{
         event::{EventReader, EventWriter},
-        schedule::IntoSystemConfigs,
-        system::{Query, ResMut, Resource},
+        resource::Resource,
+        schedule::IntoScheduleConfigs,
+        system::{Query, ResMut},
     },
     prelude::Res,
     time::{Time, Timer, TimerMode},
@@ -87,7 +88,7 @@ pub fn autoref(
                 .as_ref()
                 .is_some_and(|timer| timer.finished())
             {
-                game_controller_commands.send(GameControllerCommand::SetGameState(GameState::Set));
+                game_controller_commands.write(GameControllerCommand::SetGameState(GameState::Set));
             }
         }
         GameState::Set => {
@@ -103,7 +104,7 @@ pub fn autoref(
             {
                 match state.goal_mode {
                     GoalMode::GoToReady => {
-                        game_controller_commands.send(GameControllerCommand::Goal(scoring_team));
+                        game_controller_commands.write(GameControllerCommand::Goal(scoring_team));
                         ball.state = None;
                     }
                     GoalMode::ReturnBall => {

crates/bevyhavior_simulator/src/autoref.rs

@@ -33,7 +33,7 @@ fn startup(
     ] {
         commands.spawn(Robot::new(number));
     }
-    game_controller_commands.send(GameControllerCommand::SetGameState(GameState::Ready));
+    game_controller_commands.write(GameControllerCommand::SetGameState(GameState::Ready));
 }
 
 fn update(
@@ -44,7 +44,7 @@ fn update(
     mut ball: ResMut<BallResource>,
 ) {
     if time.ticks() == 5000 {
-        game_controller_commands.send(GameControllerCommand::SetSubState(
+        game_controller_commands.write(GameControllerCommand::SetSubState(
             Some(SubState::CornerKick),
             Team::Opponent,
             None,
@@ -58,10 +58,10 @@ fn update(
     }
     if game_controller.state.hulks_team.score > 0 {
         println!("Done");
-        exit.send(AppExit::Success);
+        exit.write(AppExit::Success);
     }
     if time.ticks() >= 20_000 {
         println!("No goal was scored :(");
-        exit.send(AppExit::from_code(1));
+        exit.write(AppExit::from_code(1));
     }
 }

crates/bevyhavior_simulator/src/bin/ball_search.rs

@@ -33,7 +33,7 @@ fn startup(
     ] {
         commands.spawn(Robot::new(number));
     }
-    game_controller_commands.send(GameControllerCommand::SetGameState(GameState::Ready));
+    game_controller_commands.write(GameControllerCommand::SetGameState(GameState::Ready));
 }
 
 fn update(time: Res<Time<Ticks>>, mut ball: ResMut<BallResource>, mut exit: EventWriter<AppExit>) {
@@ -59,6 +59,6 @@ fn update(time: Res<Time<Ticks>>, mut ball: ResMut<BallResource>, mut exit: Even
     }
 
     if time.ticks() == 30000 {
-        exit.send(AppExit::Success);
+        exit.write(AppExit::Success);
     }
 }

crates/bevyhavior_simulator/src/bin/defender_positioning.rs

@@ -33,7 +33,7 @@ fn startup(
     ] {
         commands.spawn(Robot::new(number));
     }
-    game_controller_commands.send(GameControllerCommand::SetGameState(GameState::Ready));
+    game_controller_commands.write(GameControllerCommand::SetGameState(GameState::Ready));
 }
 
 /// Allows for checks to run during the scenario such that it can be decided whether the scenario passes or fails.
@@ -56,17 +56,17 @@ fn update(
     // Scenarios can pass if a certain condition is met
     if game_controller.state.hulks_team.score > 0 {
         println!("Done");
-        exit.send(AppExit::Success);
+        exit.write(AppExit::Success);
     }
     // Or fail based on a certain condition, such as if scoring takes too long
     if time.ticks() >= 20_000 {
         println!("No goal was scored :(");
-        exit.send(AppExit::from_code(1));
+        exit.write(AppExit::from_code(1));
     }
     // Based on time or other conditions you can modify the game state
     if time.ticks() == 6000 {
         // Set substate
-        game_controller_commands.send(GameControllerCommand::SetSubState(
+        game_controller_commands.write(GameControllerCommand::SetSubState(
             Some(SubState::PushingFreeKick),
             Team::Opponent,
             Some(PlayerNumber::Four),

crates/bevyhavior_simulator/src/bin/demonstration.rs

@@ -27,7 +27,7 @@ fn startup(
     for number in [PlayerNumber::One, PlayerNumber::Seven] {
         commands.spawn(Robot::new(number));
     }
-    game_controller_commands.send(GameControllerCommand::SetGameState(GameState::Ready));
+    game_controller_commands.write(GameControllerCommand::SetGameState(GameState::Ready));
 }
 
 fn update(
@@ -54,7 +54,7 @@ fn update(
             PlayerNumber::Six,
             PlayerNumber::Seven,
         ] {
-            game_controller_commands.send(GameControllerCommand::Penalize(
+            game_controller_commands.write(GameControllerCommand::Penalize(
                 player_numer,
                 penalty,
                 Team::Hulks,
@@ -76,10 +76,10 @@ fn update(
         == Some(Action::Search)
     {
         println!("Keeper tried to enter ball search");
-        exit.send(AppExit::from_code(1));
+        exit.write(AppExit::from_code(1));
     }
     if time.ticks() >= 10_000 {
         println!("Done");
-        exit.send(AppExit::Success);
+        exit.write(AppExit::Success);
     }
 }

crates/bevyhavior_simulator/src/bin/deny_keeper_ballsearch.rs

@@ -32,7 +32,7 @@ fn startup(
         robot.simulator_parameters.ball_timeout_factor = 0.001;
         commands.spawn(robot);
     }
-    game_controller_commands.send(GameControllerCommand::SetGameState(GameState::Ready));
+    game_controller_commands.write(GameControllerCommand::SetGameState(GameState::Ready));
 }
 
 fn update(
@@ -47,10 +47,10 @@ fn update(
     }
     if game_controller.state.hulks_team.score > 0 {
         println!("Done");
-        exit.send(AppExit::Success);
+        exit.write(AppExit::Success);
     }
     if time.ticks() >= 20_000 {
         println!("No goal was scored :(");
-        exit.send(AppExit::from_code(1));
+        exit.write(AppExit::from_code(1));
     }
 }

crates/bevyhavior_simulator/src/bin/flickering_ball.rs

@@ -30,7 +30,7 @@ fn startup(
     ] {
         commands.spawn(Robot::new(number));
     }
-    game_controller_commands.send(GameControllerCommand::SetGameState(GameState::Ready));
+    game_controller_commands.write(GameControllerCommand::SetGameState(GameState::Ready));
 }
 
 fn update(
@@ -40,25 +40,25 @@ fn update(
     mut exit: EventWriter<AppExit>,
 ) {
     if time.ticks() == 3000 {
-        game_controller_commands.send(GameControllerCommand::SetSubState(
+        game_controller_commands.write(GameControllerCommand::SetSubState(
             Some(SubState::GoalKick),
             Team::Hulks,
             None,
         ));
     }
     if time.ticks() == 5000 {
-        game_controller_commands.send(GameControllerCommand::SetSubState(
+        game_controller_commands.write(GameControllerCommand::SetSubState(
             Some(SubState::GoalKick),
             Team::Opponent,
             None,
         ));
     }
     if game_controller.state.hulks_team.score > 0 {
         println!("Done");
-        exit.send(AppExit::Success);
+        exit.write(AppExit::Success);
     }
     if time.ticks() >= 10_000 {
         println!("No goal was scored :(");
-        exit.send(AppExit::from_code(1));
+        exit.write(AppExit::from_code(1));
     }
 }

crates/bevyhavior_simulator/src/bin/goal_kicks.rs

@@ -30,7 +30,7 @@ fn startup(
     ] {
         commands.spawn(Robot::new(number));
     }
-    game_controller_commands.send(GameControllerCommand::SetGameState(GameState::Ready));
+    game_controller_commands.write(GameControllerCommand::SetGameState(GameState::Ready));
 }
 
 fn update(
@@ -40,10 +40,10 @@ fn update(
 ) {
     if game_controller.state.hulks_team.score > 0 {
         println!("Done");
-        exit.send(AppExit::Success);
+        exit.write(AppExit::Success);
     }
     if time.ticks() >= 10_000 {
         println!("No goal was scored :(");
-        exit.send(AppExit::from_code(1));
+        exit.write(AppExit::from_code(1));
     }
 }