feat(sim): spatialize world audio

ros2_ws/src/brain/brain_client/brain_client/nodes/brain_client_node.py

@@ -426,7 +426,6 @@ def _on_environment_speech(self, payload: dict) -> None:
         except (KeyError, ValueError) as exc:
             self.get_logger().warning(f"Ignoring invalid environment speech request: {exc}")
             return
-
         shown = threading.Event()
 
         def show() -> None:
@@ -449,6 +448,7 @@ def deliver(_success: bool) -> None:
             on_start=show,
             on_done=deliver,
             protected=True,  # another character's line: agent flushes must not cancel it
+            audio_source=payload.get("source"),
         )
         if not queued:
             deliver(False)

ros2_ws/src/brain/brain_client/brain_client/transport/tts.py

@@ -8,6 +8,7 @@
 
 import base64
 import io
+import json
 import queue
 import struct
 import subprocess
@@ -34,6 +35,7 @@ class _Utterance:
     on_done: Callable[[bool], None] | None
     reply_id: str | None  # sentences of one streamed reply share an id
     protected: bool  # never flushed (environment speech: not our backlog)
+    audio_source: str | None = "robot"  # which simulated body speaks; None means nowhere in particular
 
 
 def _survives_flush(item: _Utterance, playing_reply_id: str | None) -> bool:
@@ -160,6 +162,7 @@ def speak_text(
         text: str,
         voice_config: dict[str, Any] | None = None,
         on_start: Callable[[], None] | None = None,
+        audio_source: str | None = "robot",
     ) -> bool:
         """
         Convert text to speech and play it.
@@ -168,6 +171,7 @@ def speak_text(
             text: Text to speak
             voice_config: Optional voice configuration override
             on_start: Called once the first audio reaches the speaker
+            audio_source: Simulated body the sound comes from (sim only; the webapp fades it by camera distance)
 
         Returns:
             True if speech was successfully generated and played, False otherwise
@@ -201,7 +205,7 @@ def speak_text(
             }
 
             if self._simulator_mode and self.tts_audio_pub is not None:
-                success = self._synthesize_to_topic(text, voice, t_start, on_start)
+                success = self._synthesize_to_topic(text, voice, t_start, on_start, audio_source)
             else:
                 success = self._synthesize_to_aplay(text, voice, t_start, on_start)
         except Exception as e:
@@ -348,8 +352,9 @@ def _synthesize_to_topic(
         voice: dict[str, Any],
         t_start: float,
         on_start: Callable[[], None] | None = None,
+        audio_source: str | None = "robot",
     ) -> bool:
-        """Synthesize the full clip and publish it (base64 WAV) on /tts/audio.
+        """Synthesize the full clip and publish it (JSON: base64 WAV + source) on /tts/audio.
 
         The sim container has no audio device, so the webapp is the speaker. We
         collect the whole clip (utterances are short) and publish it once.
@@ -370,7 +375,7 @@ def _synthesize_to_topic(
             return False
 
         wav = _finalize_wav(bytes(buf))
-        self._publish_audio(wav)
+        self._publish_audio(wav, audio_source)
         if on_start is not None:
             on_start()
         # Publishing the full clip is the beginning of playback, not the end.
@@ -386,14 +391,13 @@ def _synthesize_to_topic(
         )
         return True
 
-    def _publish_audio(self, wav: bytes) -> None:
-        """Publish one already-finalized clip on /tts/audio as base64 WAV."""
+    def _publish_audio(self, wav: bytes, audio_source: str | None) -> None:
         if self.tts_audio_pub is None or not wav:
             return
         from std_msgs.msg import String
 
-        payload = base64.b64encode(wav).decode("ascii")
-        self.tts_audio_pub.publish(String(data=payload))
+        payload = {"audio": base64.b64encode(wav).decode("ascii"), "source": audio_source}
+        self.tts_audio_pub.publish(String(data=json.dumps(payload, separators=(",", ":"))))
 
     def speak_text_async(
         self,
@@ -404,6 +408,7 @@ def speak_text_async(
         on_done: Callable[[bool], None] | None = None,
         reply_id: str | None = None,
         protected: bool = False,
+        audio_source: str | None = "robot",
     ) -> bool:
         """
         Queue text to be spoken. Utterances play in order, one at a time;
@@ -440,7 +445,9 @@ def speak_text_async(
                 self._speech_queue.extend(kept)
             queued = len(self._speech_queue) < self._speech_queue_maxlen
             if queued:
-                self._speech_queue.append(_Utterance(text, voice_config, on_start, on_done, reply_id, protected))
+                self._speech_queue.append(
+                    _Utterance(text, voice_config, on_start, on_done, reply_id, protected, audio_source)
+                )
                 self._speech_cv.notify()
         if not queued:
             self.logger.warning(f"🔇 Speech queue full, dropping: '{text[:60]}'")
@@ -516,12 +523,12 @@ def _speech_loop(self):
             # reply's flush spares siblings of speech nobody has heard, and they
             # play ahead of the newer answer.
             take_floor = self._floor_taken_on_start(item.reply_id, self._once(item.on_start))
-            success = self.speak_text(item.text, item.voice_config, take_floor)
+            success = self.speak_text(item.text, item.voice_config, take_floor, item.audio_source)
             if not success:
                 self._set_playing_reply(None)
                 self.logger.info("🔄 Retrying TTS after 1 second...")
                 time.sleep(1)
-                success = self.speak_text(item.text, item.voice_config, take_floor)
+                success = self.speak_text(item.text, item.voice_config, take_floor, item.audio_source)
             if not success:
                 self._set_playing_reply(None)
                 self._drop_queued_reply(item.reply_id)

ros2_ws/src/brain/brain_client/test/test_tts.py

@@ -46,7 +46,7 @@ def test_failed_reply_no_longer_holds_the_floor(monkeypatch):
     handler._speech_cv = threading.Condition()
     handler._playing_reply_id = None
     handler.logger = SimpleNamespace(info=lambda _message: None, error=lambda _message: None)
-    handler.speak_text = lambda _text, _voice, _on_start=None: False
+    handler.speak_text = lambda _text, _voice, _on_start=None, _audio_source="robot": False
     monkeypatch.setattr("brain_client.transport.tts.time.sleep", lambda _seconds: None)
 
     handler._speech_loop()
@@ -83,7 +83,7 @@ def enqueue_new_reply():
         assert handler.speak_text_async("new reply", replace_pending=True, reply_id="new-reply")
         handler._speech_queue.append(None)
 
-    def speak(text, _voice, on_start=None):
+    def speak(text, _voice, on_start=None, _audio_source="robot"):
         spoken.append(text)
         if text == "failed first":
             attempt = spoken.count(text)

ros2_ws/src/mars_bot/mars_sim_driver/mars_sim_driver/challenges.py

@@ -304,6 +304,7 @@ class EnvironmentReply:
     speaker: str
     text: str
     voice_id: str
+    source: str | None = None
 
 
 @dataclass
@@ -697,6 +698,8 @@ def tick(
                                     "text": reply.text,
                                     "voice_id": reply.voice_id,
                                 }
+                                if reply.source:
+                                    payload["source"] = reply.source
                             elif isinstance(reply, str):
                                 payload = {"sender": "user", "text": reply, "timestamp": time.time()}
                             else:

sim/challenges/40_household_orders/runtime.py

@@ -179,19 +179,23 @@ def update(self, state: WorldState, events: list[dict]) -> RuntimeResult:
                         resident.name,
                         f"I'm {resident.name} and I want {resident.order} Please repeat the complete order back to me.",
                         resident.voice_id,
+                        resident.prop,
                     )
                 )
                 continue
             if self._matches(resident, event["text"]):
                 self._confirmed.add(resident.id)
                 result.events.append({"type": "resident_order_confirmed", "resident": resident.id})
-                result.replies.append(EnvironmentReply(resident.name, "That's correct. Thank you.", resident.voice_id))
+                result.replies.append(
+                    EnvironmentReply(resident.name, "That's correct. Thank you.", resident.voice_id, resident.prop)
+                )
             else:
                 result.replies.append(
                     EnvironmentReply(
                         resident.name,
                         f"Not quite. I want {resident.order} Please repeat the complete order back to me.",
                         resident.voice_id,
+                        resident.prop,
                     )
                 )
         return result

sim/viewer/src/props.ts

@@ -276,6 +276,15 @@ export class PropLibrary {
     return [...this.roots.values()].filter((r) => r.visible);
   }
 
+  /** Live world positions keyed by the authoritative prop names. */
+  get audioSourcePositions(): Record<string, [number, number, number]> {
+    const positions: Record<string, [number, number, number]> = {};
+    for (const [name, root] of this.roots) {
+      if (root.visible) positions[name] = [root.position.x, root.position.y, root.position.z];
+    }
+    return positions;
+  }
+
   private buildPlacementPreview(name: string): PlacementPreview | undefined {
     const info = this.info.get(name);
     if (!info) return undefined;

sim/viewer/src/scene.ts

@@ -154,6 +154,11 @@ const TOP_FALLBACK_HEIGHT_M = 12;
 // Robot-mounted camera views: frames, axis conventions, FOV and near plane
 // match the driver's cameras (mars_sim_driver.core's CAMERAS).
 export type CameraView = "orbit" | "main" | "arm";
+export interface SimAudioPerspective {
+  view: CameraView;
+  listener: [number, number, number];
+  sources: Record<string, [number, number, number]>;
+}
 // Track mars_sim_driver/constants.py: per-camera FOVs matching what the
 // driver renders (the head and wrist are different physical lenses), so the
 // operator's preview frames what the robot consumes. main is the head's real
@@ -889,6 +894,21 @@ export class SimScene {
     this.applyControlsEnabled();
   }
 
+  /** Snapshot from the exact scene state used by the primary render. */
+  audioPerspective(): SimAudioPerspective {
+    const activeCamera =
+      (this.activeView !== "orbit" ? this.robotCameras.get(this.activeView) : undefined) ?? this.camera;
+    const listener = activeCamera.getWorldPosition(new THREE.Vector3());
+    return {
+      view: this.activeView,
+      listener: [listener.x, listener.y, listener.z],
+      sources: {
+        robot: [this.robotRoot.position.x, this.robotRoot.position.y, this.robotRoot.position.z],
+        ...this.props.audioSourcePositions,
+      },
+    };
+  }
+
   /** While a placement drag owns the pointer the orbit controls stay off,
    * even in the orbit view -- otherwise aiming a prop also spins the camera.
    * Applied immediately: a flag that is only consulted by setView() would not

sim/viewer/src/simStage.ts

@@ -31,6 +31,7 @@ const MIN_FRAME_MS = 1000 / 62;
 // are a contract with webapp/js/agent/challengePanel.js.
 const PANEL_OPEN_EVENT = "innate:panel-open";
 const PANEL_ID = "sim-scene-setup";
+const AUDIO_PERSPECTIVE_EVENT = "innate:sim-audio-perspective";
 
 const VIEW_FOR: Record<string, CameraView> = { main: "main", arm: "arm", orbit: "orbit" };
 const ROTATION_DRAG_PX = 6;
@@ -555,6 +556,9 @@ export function createSimStage(
     cancelAnimationFrame(raf);
     raf = 0;
   };
+  const clearAudioPerspective = () => {
+    document.dispatchEvent(new CustomEvent(AUDIO_PERSPECTIVE_EVENT, { detail: null }));
+  };
 
   const loop = (now: number) => {
     raf = requestAnimationFrame(loop);
@@ -581,6 +585,7 @@ export function createSimStage(
     // ...then the primary view full-frame on top.
     scene.setView(VIEW_FOR[session.primaryCamera] ?? "orbit");
     scene.render();
+    document.dispatchEvent(new CustomEvent(AUDIO_PERSPECTIVE_EVENT, { detail: scene.audioPerspective() }));
     frame++;
 
     if (perfEl) {
@@ -685,6 +690,7 @@ export function createSimStage(
     detach() {
       attached = false;
       stopLoop();
+      clearAudioPerspective();
       // The agent page lifts this into its own layout; take it back before
       // that page clears its DOM.
       wrap.appendChild(debugStack);
@@ -699,6 +705,7 @@ export function createSimStage(
       unsubscribeProps();
       unsubscribeEnvironment();
       stopLoop();
+      clearAudioPerspective();
       observer.disconnect();
       longTaskObserver?.disconnect();
       window.removeEventListener("pointerup", finishDrop);

tests/test_challenge_runtime.py

@@ -51,7 +51,9 @@ def update(self, _state, events) -> RuntimeResult:
                 continue
             resident = event["text"]
             result.events.append({"type": "confirmed", "resident": resident})
-            result.replies.append(EnvironmentReply(resident.title(), "Confirmed", f"voice-{resident}"))
+            result.replies.append(
+                EnvironmentReply(resident.title(), "Confirmed", f"voice-{resident}", f"resident-{resident}")
+            )
         return result
 
 
@@ -126,6 +128,7 @@ def test_environment_reply_is_a_speech_request_the_brain_voices(tmp_path):
     assert payload["speaker"] == "A"
     assert payload["sender"] == "environment_speech"
     assert payload["voice_id"] == "voice-a"
+    assert payload["source"] == "resident-a"
 
 
 def test_restart_invalidates_queued_and_dequeued_replies(tmp_path):

webapp/index.html

@@ -30,6 +30,7 @@
     <link rel="modulepreload" href="/js/pressActivate.js" />
     <link rel="modulepreload" href="/js/ttsAudio.js" />
     <link rel="modulepreload" href="/js/motorSoundAudio.js" />
+    <link rel="modulepreload" href="/js/simSpatialAudio.js" />
     <link rel="modulepreload" href="/js/micAudioState.js" />
     <link rel="modulepreload" href="/js/agent/main.js" />
     <link rel="modulepreload" href="/js/agent/agentPanel.js" />

webapp/js/motorSoundAudio.js

@@ -6,6 +6,7 @@
 
 import { ros } from "./rosClient.js";
 import { base64ToBytes, isRobotAudioSpeaker } from "./ttsAudio.js";
+import { followSimAudioSource } from "./simSpatialAudio.js";
 
 const TOPIC = "/motor_sound/audio";
 const LEAD_S = 0.12;
@@ -14,6 +15,8 @@ const MAX_QUEUED_S = 0.5;
 let started = false;
 /** @type {AudioContext | null} */
 let context = null;
+/** @type {GainNode | null} */
+let output = null;
 let nextAt = 0;
 
 export function initMotorSoundAudio() {
@@ -25,7 +28,12 @@ export function initMotorSoundAudio() {
   const unlock = () => {
     if (!context) {
       const Context = window.AudioContext || /** @type {any} */ (window).webkitAudioContext;
-      if (Context) context = new Context();
+      if (Context) {
+        context = new Context();
+        output = context.createGain();
+        output.connect(context.destination);
+        followSimAudioSource("robot", (gain) => output?.gain.setTargetAtTime(gain, context?.currentTime ?? 0, 0.05));
+      }
     }
     void context?.resume();
   };
@@ -40,27 +48,30 @@ export function initMotorSoundAudio() {
   });
 
   ros.subscribe(TOPIC, (msg) => {
-    if (!context || document.visibilityState === "hidden" || !isRobotAudioSpeaker() || typeof msg?.data !== "string") {
+    if (!output || document.visibilityState === "hidden" || !isRobotAudioSpeaker() || typeof msg?.data !== "string") {
       return;
     }
+    const out = output;
+    const ctx = /** @type {AudioContext} */ (out.context);
     let payload;
     try {
       payload = JSON.parse(msg.data);
     } catch {
       return; // a malformed audio packet must not disrupt the shell
     }
-    if (context.state === "running") {
-      schedule(context, payload);
+    if (ctx.state === "running") {
+      schedule(out, payload);
       return;
     }
-    void context.resume().then(() => {
-      if (context && document.visibilityState !== "hidden") schedule(context, payload);
+    void ctx.resume().then(() => {
+      if (document.visibilityState !== "hidden") schedule(out, payload);
     }).catch(() => {});
   }, undefined, "std_msgs/msg/String");
 }
 
-/** @param {AudioContext} ctx @param {{ sample_rate?: unknown, pcm?: unknown }} payload */
-function schedule(ctx, payload) {
+/** @param {GainNode} out @param {{ sample_rate?: unknown, pcm?: unknown }} payload */
+function schedule(out, payload) {
+  const ctx = /** @type {AudioContext} */ (out.context);
   const rate = Number(payload?.sample_rate);
   if (!Number.isFinite(rate) || rate < 8_000 || rate > 192_000 || typeof payload?.pcm !== "string") return;
   let bytes;
@@ -81,7 +92,7 @@ function schedule(ctx, payload) {
   for (let i = 0; i < samples.length; i++) channel[i] = samples[i] / 32768;
   const source = ctx.createBufferSource();
   source.buffer = buffer;
-  source.connect(ctx.destination);
+  source.connect(out);
   source.start(nextAt);
   nextAt += buffer.duration;
 }