fix(sim): play Mad Mars sound in browser

Author: axelpey

ros2_ws/src/mars_bot/mars_control/launch/app.sim.launch.py

@@ -1,12 +1,16 @@
 # SPDX-License-Identifier: Apache-2.0
 # Copyright (c) 2026 Innate Inc
+import os
+
+from ament_index_python.packages import get_package_share_directory
 from launch import LaunchDescription
 from launch_ros.actions import Node
 from mars_bringup.config_loader import innate_os_root, settings_params
 
 
 def generate_launch_description():
     data_directory = str(innate_os_root() / "data")
+    motor_sound_config = os.path.join(get_package_share_directory("mars_control"), "config", "motor_sound.yaml")
 
     # Default hardware revision for new robots
     default_hardware_revision = "R6"
@@ -25,4 +29,14 @@ def generate_launch_description():
         ],
     )
 
-    return LaunchDescription([app_node])
+    # The container has no speaker. Stream the same configured synth as PCM;
+    # the simulator webapp is the audio device.
+    motor_sound_node = Node(
+        package="mars_control",
+        executable="motor_sound.py",
+        name="motor_sound",
+        parameters=[motor_sound_config, *settings_params(), {"motor_sound.browser_audio_topic": "/motor_sound/audio"}],
+        output="screen",
+    )
+
+    return LaunchDescription([app_node, motor_sound_node])

ros2_ws/src/mars_bot/mars_control/mars_control/motor_sound.py

@@ -20,11 +20,12 @@
 * throttle demand is the commanded acceleration, which is what makes
   accelerating sound different from cruising at the same speed.
 
-Audio goes to the ALSA ``default`` device, which ``config/alsa/asound.conf``
-defines as dmix + softvol. dmix is what lets this run alongside TTS instead of
-fighting it for the device.
+On hardware, audio goes to the ALSA ``default`` device, which
+``config/alsa/asound.conf`` defines as dmix + softvol. In simulation the same
+synth is streamed as PCM to the browser because the container has no speaker.
 """
 
+import base64
 import json
 import time
 
@@ -54,8 +55,8 @@
 LIVE_PARAMS = frozenset({"enabled", "volume", "idle_when_stopped", "only_in_mad_mode", "reference_acceleration"})
 """Settable in place. Every other motor_sound.* parameter is baked into the
 synth at construction, so applying one live means rebuilding it."""
-STREAM_PARAMS = frozenset({"sample_rate", "blocksize", "device"})
-"""Baked into the audio stream, which opens once at boot. A live set is
+STREAM_PARAMS = frozenset({"sample_rate", "blocksize", "device", "browser_audio_topic"})
+"""Baked into the audio output, which opens once at boot. A live set is
 rejected outright rather than pretending to apply."""
 
 
@@ -98,8 +99,14 @@ def __init__(self):
         self.create_timer(0.1, self._update_drive)
 
         self._update_drive()
-        self._stream = self._open_stream(params)
-        if self._stream is not None and not self._only_in_mad_mode:
+        browser_topic = str(params["browser_audio_topic"].value)
+        self._browser_audio = self.create_publisher(String, browser_topic, 2) if browser_topic else None
+        self._stream = None if self._browser_audio is not None else self._open_stream(params)
+        if self._browser_audio is not None:
+            self._browser_frames = max(int(params["blocksize"].value), int(params["sample_rate"].value) // 10)
+            self.create_timer(self._browser_frames / int(params["sample_rate"].value), self._publish_browser_audio)
+            self.get_logger().info(f"motor sound streaming to {browser_topic}")
+        if (self._stream is not None or self._browser_audio is not None) and not self._only_in_mad_mode:
             self._synth.trigger_startup()
 
     def _declare_parameters(self):
@@ -144,6 +151,8 @@ def _declare_parameters(self):
                 ("motor_sound.sample_rate", 48000),
                 ("motor_sound.blocksize", 512),
                 ("motor_sound.device", ""),
+                # Non-empty only in simulation: the browser is its speaker.
+                ("motor_sound.browser_audio_topic", ""),
             ],
         )
 
@@ -233,19 +242,35 @@ def _audio_callback(self, outdata, frames, _time_info, _status):
         escaping here silently aborts the stream, so it swallows everything and
         outputs silence instead."""
         try:
-            synth = self._synth
-            mono = synth.render(frames)
-            retiring = self._retiring
-            # The identity check covers the instant between _swap_synth parking
-            # the old synth and rebinding _synth: never fade a voice under itself.
-            if retiring is not None and retiring is not synth:
-                self._retiring = None
-                mono = mono + retiring.render(frames) * np.linspace(1.0, 0.0, frames)
+            mono = self._render(frames)
             outdata[:, 0] = mono
             outdata[:, 1] = mono
         except Exception:
             outdata.fill(0.0)
 
+    def _render(self, frames: int) -> np.ndarray:
+        synth = self._synth
+        mono = synth.render(frames)
+        retiring = self._retiring
+        # The identity check covers the instant between _swap_synth parking
+        # the old synth and rebinding _synth: never fade a voice under itself.
+        if retiring is not None and retiring is not synth:
+            self._retiring = None
+            mono = mono + retiring.render(frames) * np.linspace(1.0, 0.0, frames)
+        return mono
+
+    def _publish_browser_audio(self):
+        """Send the same synth to the simulator UI as signed 16-bit mono PCM."""
+        mono = self._render(self._browser_frames)
+        if not np.any(mono):
+            return
+        pcm = (np.clip(mono, -1.0, 1.0) * 32767.0).astype("<i2", copy=False)
+        payload = {
+            "sample_rate": self._synth.sample_rate,
+            "pcm": base64.b64encode(pcm.tobytes()).decode("ascii"),
+        }
+        self._browser_audio.publish(String(data=json.dumps(payload, separators=(",", ":"))))
+
     def _on_robot_info(self, msg: String):
         try:
             mad = is_mad_scale(json.loads(msg.data))

webapp/index.html

@@ -29,6 +29,8 @@
     <link rel="modulepreload" href="/js/armReboot.js" />
     <link rel="modulepreload" href="/js/pressActivate.js" />
     <link rel="modulepreload" href="/js/ttsAudio.js" />
+    <link rel="modulepreload" href="/js/motorSoundAudio.js" />
+    <link rel="modulepreload" href="/js/pcmAudioPlayer.js" />
     <link rel="modulepreload" href="/js/micAudioState.js" />
     <link rel="modulepreload" href="/js/agent/main.js" />
     <link rel="modulepreload" href="/js/agent/agentPanel.js" />

webapp/js/motorSoundAudio.js

@@ -0,0 +1,54 @@
+// @ts-check
+// The physical robot plays its motor voice through ALSA. Simulation publishes
+// the identical synth as PCM because its container has no audio device.
+
+import { ros } from "./rosClient.js";
+import { PcmAudioPlayer } from "./pcmAudioPlayer.js";
+import { isRobotAudioSpeaker } from "./ttsAudio.js";
+
+const TOPIC = "/motor_sound/audio";
+let started = false;
+/** @type {PcmAudioPlayer | null} */
+let player = null;
+
+export function initMotorSoundAudio() {
+  if (started) return;
+  started = true;
+
+  // Creating/resuming the context inside the operator's first gesture satisfies
+  // browser autoplay rules. Until then chunks are deliberately dropped.
+  const unlock = () => {
+    if (!player) {
+      const Context = window.AudioContext || /** @type {any} */ (window).webkitAudioContext;
+      if (Context) player = new PcmAudioPlayer(new Context());
+    }
+    void player?.context.resume();
+  };
+  window.addEventListener("pointerdown", unlock, { once: true, capture: true });
+  window.addEventListener("keydown", unlock, { once: true, capture: true });
+  document.addEventListener("visibilitychange", () => {
+    if (document.visibilityState !== "hidden" || !player) return;
+    player.reset();
+    void player.context.suspend().catch(() => {});
+  });
+
+  ros.subscribe(TOPIC, (msg) => {
+    if (!player || document.visibilityState === "hidden" || !isRobotAudioSpeaker() || typeof msg?.data !== "string") {
+      return;
+    }
+    let payload;
+    try {
+      payload = JSON.parse(msg.data);
+    } catch {
+      // A malformed audio packet should not disrupt the shell.
+      return;
+    }
+    if (player.context.state === "running") {
+      player.push(payload);
+      return;
+    }
+    void player.context.resume().then(() => {
+      if (document.visibilityState !== "hidden") player?.push(payload);
+    }).catch(() => {});
+  }, undefined, "std_msgs/msg/String");
+}

webapp/js/pcmAudioPlayer.js

@@ -0,0 +1,61 @@
+// @ts-check
+// Gap-resistant scheduler for the simulator's short mono PCM chunks.
+
+const LEAD_S = 0.12;
+const MAX_QUEUED_S = 0.5;
+
+export class PcmAudioPlayer {
+  /** @param {AudioContext} context */
+  constructor(context) {
+    this.context = context;
+    this.nextAt = 0;
+    /** @type {Set<AudioBufferSourceNode>} */
+    this.sources = new Set();
+  }
+
+  /** @param {{ sample_rate?: unknown, pcm?: unknown }} payload */
+  push(payload) {
+    const rate = Number(payload?.sample_rate);
+    if (!Number.isFinite(rate) || rate < 8_000 || rate > 192_000 || typeof payload?.pcm !== "string") return false;
+
+    let bytes;
+    try {
+      const raw = atob(payload.pcm);
+      if (!raw.length || raw.length % 2) return false;
+      bytes = new Uint8Array(raw.length);
+      for (let i = 0; i < raw.length; i++) bytes[i] = raw.charCodeAt(i);
+    } catch {
+      return false;
+    }
+
+    const samples = new Int16Array(bytes.buffer);
+    const buffer = this.context.createBuffer(1, samples.length, rate);
+    const channel = buffer.getChannelData(0);
+    for (let i = 0; i < samples.length; i++) channel[i] = samples[i] / 32768;
+
+    const now = this.context.currentTime;
+    if (this.nextAt > now + MAX_QUEUED_S) return false;
+    if (this.nextAt < now) this.nextAt = now + LEAD_S;
+    const source = this.context.createBufferSource();
+    source.buffer = buffer;
+    source.connect(this.context.destination);
+    this.sources.add(source);
+    source.onended = () => this.sources.delete(source);
+    source.start(this.nextAt);
+    this.nextAt += buffer.duration;
+    return true;
+  }
+
+  /** Stop queued sound immediately; used when a tab is backgrounded. */
+  reset() {
+    for (const source of this.sources) {
+      try {
+        source.stop();
+      } catch {
+        // It may have ended between the Set iteration and stop().
+      }
+    }
+    this.sources.clear();
+    this.nextAt = 0;
+  }
+}

webapp/js/shell.js

@@ -6,6 +6,7 @@
 
 import { ros } from "./rosClient.js";
 import { initTtsAudio } from "./ttsAudio.js";
+import { initMotorSoundAudio } from "./motorSoundAudio.js";
 import { getConfig } from "./config.js";
 import { sharedAgentState } from "./teleop/agentState.js";
 import { createAgentIndicator } from "./agentIndicator.js";
@@ -169,6 +170,7 @@ export function initShell(navigate) {
 
   // Play robot speech (/tts/audio) regardless of which page is open; idempotent.
   initTtsAudio();
+  initMotorSoundAudio();
 
   // A running agent shows a top-center "running" pill, linking back to the Agent
   // page to take control. It's persistent (built once); setActive hides it while

webapp/js/ttsAudio.js

@@ -25,6 +25,11 @@ navigator.locks?.request("innate-tts-speaker", () => {
   return new Promise(() => {}); // hold until this tab closes
 });
 
+/** Shared by continuous simulator audio so one tab owns every robot sound. */
+export function isRobotAudioSpeaker() {
+  return speaker;
+}
+
 export function initTtsAudio() {
   if (started) return;
   started = true;

webapp/tests/pcmAudioPlayer.test.js

@@ -0,0 +1,46 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright (c) 2026 Innate Inc
+
+import assert from "node:assert/strict";
+import { PcmAudioPlayer } from "../js/pcmAudioPlayer.js";
+
+const starts = [];
+const stops = [];
+let decoded = null;
+const context = {
+  currentTime: 10,
+  destination: {},
+  createBuffer(_channels, length, sampleRate) {
+    const channel = new Float32Array(length);
+    decoded = channel;
+    return { duration: length / sampleRate, getChannelData: () => channel, channel };
+  },
+  createBufferSource() {
+    return {
+      connect() {},
+      start: (at) => starts.push(at),
+      stop: () => stops.push(true),
+      onended: null,
+      buffer: null,
+    };
+  },
+};
+
+const pcm = Buffer.from(new Int16Array([-32768, 0, 32767]).buffer).toString("base64");
+const player = new PcmAudioPlayer(/** @type {any} */ (context));
+assert.equal(player.push({ sample_rate: 48_000, pcm }), true);
+assert.deepEqual(starts, [10.12]);
+assert.deepEqual([...decoded], [-1, 0, 32767 / 32768]);
+assert.equal(player.nextAt, 10.12 + 3 / 48_000);
+assert.equal(player.push({ sample_rate: 0, pcm }), false);
+assert.equal(player.push({ sample_rate: 48_000, pcm: "not base64!" }), false);
+player.nextAt = context.currentTime + 1;
+assert.equal(player.push({ sample_rate: 48_000, pcm }), false);
+assert.deepEqual(starts, [10.12]);
+player.nextAt = 0;
+assert.equal(player.push({ sample_rate: 48_000, pcm }), true);
+player.reset();
+assert.equal(stops.length, 2);
+assert.equal(player.nextAt, 0);
+
+console.log("ok - simulator PCM is decoded, bounded, and stopped when playback resets");