Wake/sleep phrases; Auto-sleep after 30s of no movement; Refactor micLevelGate.ts to create a small audio buffer so the beginning of utterances aren't clipped;

Author: hello-robot

ai-gateway/constants.d.ts

@@ -80,3 +80,10 @@ export declare const MACRO_MOVE_ACTIONS: readonly [
 
 export declare const VOICE_SPEEDS: readonly ["slow", "medium", "fast"];
 export declare const VOICE_SPEED_DEFAULT: "medium";
+export declare const VOICE_SLEEP_PHRASE: "bye bye stretch";
+export declare const VOICE_WAKE_PHRASE_DISPLAY: "Hello Stretch";
+export declare const VOICE_SLEEP_PHRASE_DISPLAY: "Bye bye Stretch";
+export declare const VOICE_WAKE_PHRASE_ALT: "hello robot";
+export declare const VOICE_SLEEP_PHRASE_ALT: "bye bye robot";
+export declare const VOICE_WAKE_PHRASE_ALT_DISPLAY: "Hello Robot";
+export declare const VOICE_SLEEP_PHRASE_ALT_DISPLAY: "Bye bye Robot";

ai-gateway/constants.js

@@ -62,7 +62,13 @@ const EXECUTE_MACRO = "execute_macro";
 const VOICE_MACRO_NAMES = ["center_wrist", "stow_wrist"];
 
 /** All voice Realtime tools registered in the session payload and executed on the client. */
-const VOICE_TOOLS = [EXECUTE_BASE_MOVE, EXECUTE_JOINT_MOVE, STOP_MOTION, REPEAT_BASE_MOVE, EXECUTE_MACRO];
+const VOICE_TOOLS = [
+    EXECUTE_BASE_MOVE,
+    EXECUTE_JOINT_MOVE,
+    STOP_MOTION,
+    REPEAT_BASE_MOVE,
+    EXECUTE_MACRO,
+];
 
 // ── Base move action enums ────────────────────────────────────────────────────
 
@@ -132,6 +138,31 @@ const VOICE_SPEEDS = ["slow", "medium", "fast"];
 /** Server schema default when the model omits speed. */
 const VOICE_SPEED_DEFAULT = "medium";
 
+/** Normalized wake phrase (substring match in client transcripts). */
+const VOICE_WAKE_PHRASE = "hello stretch";
+
+/** Normalized sleep phrase (substring match in client transcripts). */
+const VOICE_SLEEP_PHRASE = "bye bye stretch";
+
+/** User-facing wake phrase label (UI, errors, model instructions). */
+const VOICE_WAKE_PHRASE_DISPLAY = "Hello Stretch";
+
+/** User-facing sleep phrase label (UI, errors, model instructions). */
+const VOICE_SLEEP_PHRASE_DISPLAY = "Bye bye Stretch";
+
+/** Alternate normalized wake phrase (substring match in client transcripts). */
+const VOICE_WAKE_PHRASE_ALT = "hello robot";
+
+/** Alternate normalized sleep phrase (substring match in client transcripts). */
+const VOICE_SLEEP_PHRASE_ALT = "bye bye robot";
+
+/** User-facing alternate wake phrase label. */
+const VOICE_WAKE_PHRASE_ALT_DISPLAY = "Hello Robot";
+
+/** User-facing alternate sleep phrase label. */
+const VOICE_SLEEP_PHRASE_ALT_DISPLAY = "Bye bye Robot";
+
+
 module.exports = {
     VOICE_DURATION_MS_MIN,
     VOICE_DURATION_MS_MAX,
@@ -150,6 +181,7 @@ module.exports = {
     REPEAT_BASE_MOVE,
     EXECUTE_MACRO,
     VOICE_MACRO_NAMES,
+
     VOICE_TOOLS,
     BASE_TRANSLATE_ACTIONS,
     BASE_ROTATE_ACTIONS,
@@ -160,4 +192,12 @@ module.exports = {
     JOINT_MOVE_ACTIONS,
     VOICE_SPEEDS,
     VOICE_SPEED_DEFAULT,
+    VOICE_WAKE_PHRASE,
+    VOICE_SLEEP_PHRASE,
+    VOICE_WAKE_PHRASE_DISPLAY,
+    VOICE_SLEEP_PHRASE_DISPLAY,
+    VOICE_WAKE_PHRASE_ALT,
+    VOICE_SLEEP_PHRASE_ALT,
+    VOICE_WAKE_PHRASE_ALT_DISPLAY,
+    VOICE_SLEEP_PHRASE_ALT_DISPLAY,
 };

ai-gateway/openai-realtime-api.js

@@ -35,6 +35,10 @@ const {
     JOINT_DISTANCE_RAD_MAX,
     EXECUTE_MACRO,
     VOICE_MACRO_NAMES,
+    VOICE_WAKE_PHRASE_DISPLAY,
+    VOICE_SLEEP_PHRASE_DISPLAY,
+    VOICE_WAKE_PHRASE_ALT_DISPLAY,
+    VOICE_SLEEP_PHRASE_ALT_DISPLAY,
 } = require("./constants");
 
 /**
@@ -68,7 +72,10 @@ function buildRealtimeVoiceSessionPayload() {
             type: "realtime",
             model: "gpt-realtime-2",
             instructions: [
-                "Don't speak to the user",
+                "Do not respond to the user. Do not speak to the user. Only call tools as instructed.",
+                // Wake/sleep phrases — do not tool-call
+                `When the user says only ${VOICE_WAKE_PHRASE_DISPLAY}, ${VOICE_WAKE_PHRASE_ALT_DISPLAY}, or similar wake greetings, do NOT call any tool.`,
+                `When the user says only ${VOICE_SLEEP_PHRASE_DISPLAY}, ${VOICE_SLEEP_PHRASE_ALT_DISPLAY}, or similar farewells, do NOT call any tool.`,
                 // ── Base move harness ────────────────────────────────────────────────────
                 "The user speaks imprecisely. Infer exactly ONE motion command per turn.",
                 `For BASE movement (forward, backward, strafe, rotate), call tool ${EXECUTE_BASE_MOVE}.`,
@@ -233,12 +240,24 @@ function buildRealtimeVoiceSessionPayload() {
             tool_choice: "auto",
             audio: {
                 input: {
+
+                    transcription: {
+                        model: "gpt-4o-mini-transcribe",
+                        language: "en",
+                    },
+                    /** Docs: https://developers.openai.com/api/docs/guides/realtime-vad#server-vad  */
                     turn_detection: {
                         type: "server_vad",
-                        threshold: 0.7,
+                        // How loud and clear incoming audio must be
+                        // before the server counts it as speech
+                        threshold: 0.5,
+                        // When VAD detects speech, OpenAI includes 300ms
+                        // of audio from before the trigger point in that turn
                         prefix_padding_ms: 300,
                         silence_duration_ms: 300,
                         create_response: true,
+                        // If user starts speaking while it will
+                        // interrupt the prior in-flight response
                         interrupt_response: true,
                     },
                 },

src/pages/operator/tsx/function_providers/ButtonFunctionProvider.tsx

@@ -1,6 +1,6 @@
 import {
-    JOINT_VELOCITIES,
     JOINT_INCREMENTS,
+    JOINT_VELOCITIES,
     ValidJoints,
     ValidJointStateDict,
 } from "shared/util";
@@ -240,9 +240,6 @@ export class ButtonFunctionProvider extends FunctionProvider {
         if (!FunctionProvider.remoteRobot) {
             return false;
         }
-        if (this.timedVoiceMoveActive) {
-            return false;
-        }
 
         const clampedMs = clampDurationMs(durationMs);
         this.stopCurrentAction(true);

src/pages/operator/tsx/function_providers/FunctionProvider.tsx

@@ -1,13 +1,13 @@
-import { RemoteRobot } from "shared/remoterobot";
 import { VelocityCommand } from "shared/commands";
+import { RemoteRobot } from "shared/remoterobot";
 import { RobotPose, ValidJoints } from "shared/util";
+import { PilotButtonPads } from "../static_components/PilotControlsToggle";
 import {
     ActionModeType,
     PilotButtonPadType,
 } from "../utils/component_definitions";
 import { clampDurationMs } from "../voice/constants";
 import { ButtonPadButton } from "./ButtonFunctionProvider";
-import { PilotButtonPads } from "../static_components/PilotControlsToggle";
 
 const x = PilotButtonPads;
 
@@ -63,6 +63,15 @@ export abstract class FunctionProvider {
         return FunctionProvider.remoteRobot !== undefined;
     }
 
+    /** True while a timed voice move or continuous velocity heartbeat is active. */
+    public isMotionActive(): boolean {
+        return (
+            this.timedVoiceMoveActive ||
+            this.velocityExecutionHeartbeat !== undefined ||
+            this.activeVelocityAction !== undefined
+        );
+    }
+
     /**
      * Move the robot to an absolute pose (e.g. a voice macro).
      * Stops any ongoing velocity or timed move, then sends a setRobotPose command.
@@ -149,7 +158,7 @@ export abstract class FunctionProvider {
      * @param jointName the joint to actuate
      * @param velocity  signed velocity in joint units/s (m/s for linear, rad/s for rotary)
      * @param durationMs how long to apply velocity (clamped internally)
-     * @returns false if a timed move is already active or no robot attached
+     * @returns false if no robot attached
      */
     public timedJointMove(
         jointName: ValidJoints,
@@ -159,9 +168,6 @@ export abstract class FunctionProvider {
         if (!FunctionProvider.remoteRobot) {
             return false;
         }
-        if (this.timedVoiceMoveActive) {
-            return false;
-        }
 
         const clampedMs = clampDurationMs(durationMs);
 

src/pages/operator/tsx/static_components/VoiceCommandAssistant.tsx

@@ -3,36 +3,39 @@
  * sits at the top of the screen. Very WIP.
  */
 
+import "operator/css/VoiceCommandAssistant.css";
 import React, {
     useCallback,
     useEffect,
     useRef,
     useState,
 } from "react";
+import { getOperatorVoiceSessionToken } from "shared/operatorVoiceSession";
 import { buttonFunctionProvider } from "..";
+import Ellipsis from "../basic_components/Ellipsis";
 import { FunctionProvider } from "../function_providers/FunctionProvider";
-import { ActionModeType } from "../utils/component_definitions";
+import type { AddToastFn } from "../layout_components/Toasts";
 import { velocityScaleForVoiceSpeed } from "../utils/action-speed-scale";
-import {
-    connectOpenAIRealtimeVoice,
-    type ActiveRealtimeVoiceSession,
-} from "../voice/realtimeSession";
-import { setVoiceMoveExecutionContext } from "../voice/executeBaseMove";
-import { getOperatorVoiceSessionToken } from "shared/operatorVoiceSession";
+import { ActionModeType } from "../utils/component_definitions";
 import {
     isVoiceToolLogLine,
     VOICE_MIC_RMS_THRESHOLD,
-    type VoiceSpeed,
+    VOICE_WAKE_PHRASE_ALT_DISPLAY,
+    VOICE_WAKE_PHRASE_DISPLAY,
     type VoiceMoveExecutionMode,
+    type VoiceSpeed,
 } from "../voice/constants";
+import { setVoiceMoveExecutionContext } from "../voice/executeBaseMove";
+import {
+    connectOpenAIRealtimeVoice,
+    type ActiveRealtimeVoiceSession,
+    type VoiceListeningState,
+} from "../voice/realtimeSession";
 import { voiceMoveFeedbackToToast, type VoiceMoveFeedback } from "../voice/voiceMoveFeedback";
 import {
     AccessibleRadioGroup,
     type AccessibleRadioOption,
 } from "./AccessibleRadioGroup";
-import Ellipsis from "../basic_components/Ellipsis";
-import type { AddToastFn } from "../layout_components/Toasts";
-import "operator/css/VoiceCommandAssistant.css";
 
 /** Explicit check to make sure operator is using
  * LAN/ngrok (LocalStorage + socket.io) and not cloud (Firebase) */
@@ -86,6 +89,8 @@ export const VoiceCommandAssistant = ({
     );
     const [voiceMoveExecutionMode, voiceMoveExecutionModeSet] =
         useState<VoiceMoveExecutionMode>("direct");
+    const [listeningState, listeningStateSet] =
+        useState<VoiceListeningState>("asleep");
 
     const executionModeLocked =
         phase === "connecting" || phase === "live";
@@ -109,6 +114,7 @@ export const VoiceCommandAssistant = ({
         statusLineSet("Connect");
         micLevelSet(0);
         micGateOpenSet(false);
+        listeningStateSet("asleep");
     }, []);
 
     useEffect(() => {
@@ -151,12 +157,14 @@ export const VoiceCommandAssistant = ({
                 onVoiceSpeedChange,
                 onVoicePressAndHoldRequired,
                 onVoiceMoveFeedback,
+                onListeningState: listeningStateSet,
                 onLog: (lg) => {
-                    if (isVoiceToolLogLine(lg)) {
-                        console.log(
-                            "[VoiceCommandAssistant] tool trace:",
-                            lg.slice(0, 240),
-                        );
+                    if (
+                        lg.includes("[WakeSleep]") ||
+                        lg.includes("user transcript") ||
+                        isVoiceToolLogLine(lg)
+                    ) {
+                        console.log("[VoiceCommandAssistant]", lg.slice(0, 240));
                     }
                 },
                 onMicLevel: (level, gateOpen) => {
@@ -192,11 +200,11 @@ export const VoiceCommandAssistant = ({
     const micGateHint =
         phase !== "live"
             ? ""
-            : micGateOpen
-                ? "Listening..."
-                : micLevel > 0.001
-                    ? "Microphone gate closed"
-                    : "Ready";
+            : listeningState === "asleep"
+                ? `Asleep — say "${VOICE_WAKE_PHRASE_DISPLAY}" or "${VOICE_WAKE_PHRASE_ALT_DISPLAY}" to wake (or tap Wake)`
+                : micGateOpen
+                    ? "Listening..."
+                    : "Awake — speak your command";
 
     const primaryButtonLabel =
         phase === "live"
@@ -359,7 +367,9 @@ export const VoiceCommandAssistant = ({
                                         transform: `scaleX(${meterFill})`,
                                         backgroundColor: micGateOpen
                                             ? "hsl(184deg 100% 50%)"
-                                            : "hsla(184, 100%, 50%, 0.45)",
+                                            : listeningState === "asleep"
+                                                ? "hsla(184, 100%, 50%, 0.2)"
+                                                : "hsla(184, 100%, 50%, 0.45)",
                                         transition: "transform 0.05s linear",
                                         willChange: "transform",
                                     }}
@@ -383,13 +393,28 @@ export const VoiceCommandAssistant = ({
                                 fontWeight: 400,
                                 fontSize: "0.82em",
                                 margin: 0,
-                                color: micGateOpen
-                                    ? "hsl(184deg 100% 50%)"
-                                    : "hsl(184deg 60% 87%)",
+                                color:
+                                    listeningState === "awake" && micGateOpen
+                                        ? "hsl(184deg 100% 50%)"
+                                        : "hsl(184deg 60% 87%)",
                             }}
                         >
                             {micGateHint}
                         </p>
+                        {listeningState === "asleep" ? (
+                            <button
+                                type="button"
+                                style={{
+                                    ...styleButton,
+                                    height: 36,
+                                    fontSize: "0.9em",
+                                    marginTop: 4,
+                                }}
+                                onClick={() => sessionRef.current?.wake()}
+                            >
+                                Wake
+                            </button>
+                        ) : null}
                     </div>
                 ) : null}
             </div>