Clarify STT model modes (#5569)

Show batch and realtime badges for Soniox, AssemblyAI, ElevenLabs, and Mistral model variants, and keep realtime aliases safe for batch transcription.

Author: ComputelessComputer

apps/desktop/src/settings/ai/stt/select.tsx

@@ -370,16 +370,44 @@ function getProviderModelMode(
   providerId: ProviderId,
   model: string,
 ): ModelEntry["mode"] {
-  if (providerId !== "soniox") {
-    return undefined;
+  if (providerId === "assemblyai") {
+    if (model === "universal-3-pro") {
+      return "batch";
+    }
+
+    if (model === "u3-rt-pro") {
+      return "realtime";
+    }
+  }
+
+  if (providerId === "elevenlabs") {
+    if (model === "scribe_v2") {
+      return "batch";
+    }
+
+    if (model === "scribe_v2_realtime") {
+      return "realtime";
+    }
   }
 
-  if (model === "stt-v5" || model === "stt-async-v5") {
-    return "batch";
+  if (providerId === "mistral") {
+    if (model === "voxtral-mini-2602" || model === "voxtral-mini-latest") {
+      return "batch";
+    }
+
+    if (model === "voxtral-mini-transcribe-realtime-2602") {
+      return "realtime";
+    }
   }
 
-  if (model === "stt-v4" || model === "stt-rt-v4") {
-    return "realtime";
+  if (providerId === "soniox") {
+    if (model === "stt-v5" || model === "stt-async-v5") {
+      return "batch";
+    }
+
+    if (model === "stt-v4" || model === "stt-rt-v4") {
+      return "realtime";
+    }
   }
 
   return undefined;

apps/desktop/src/settings/ai/stt/shared.tsx

@@ -34,11 +34,11 @@ export const displayModelId = (model: string) => {
     return "Pro (Cloud)";
   }
 
-  if (
-    model === "universal-3-pro" ||
-    model === "u3-rt-pro" ||
-    model === "universal"
-  ) {
+  if (model === "u3-rt-pro") {
+    return "Universal-3 Pro Streaming";
+  }
+
+  if (model === "universal-3-pro" || model === "universal") {
     return "Universal-3 Pro";
   }
 
@@ -62,6 +62,10 @@ export const displayModelId = (model: string) => {
     return "Solaria 1";
   }
 
+  if (model === "scribe_v2_realtime") {
+    return "Scribe V2 Realtime";
+  }
+
   if (model === "scribe_v2") {
     return "Scribe V2";
   }
@@ -78,6 +82,10 @@ export const displayModelId = (model: string) => {
     return "GPT-4o mini Transcribe";
   }
 
+  if (model === "voxtral-mini-transcribe-realtime-2602") {
+    return "Voxtral Realtime";
+  }
+
   if (model === "voxtral-mini-2602") {
     return "Voxtral Mini Transcribe 2";
   }
@@ -155,7 +163,7 @@ const _PROVIDERS = [
     badge: null,
     icon: <AssemblyAI size={12} />,
     baseUrl: "https://api.assemblyai.com",
-    models: ["universal-3-pro"],
+    models: ["universal-3-pro", "u3-rt-pro"],
     requirements: [{ kind: "requires_config", fields: ["api_key"] }],
   },
   {
@@ -207,7 +215,7 @@ const _PROVIDERS = [
     badge: null,
     icon: <ElevenLabs size={16} />,
     baseUrl: "https://api.elevenlabs.io",
-    models: ["scribe_v2"],
+    models: ["scribe_v2", "scribe_v2_realtime"],
     requirements: [{ kind: "requires_config", fields: ["api_key"] }],
   },
   {
@@ -217,7 +225,7 @@ const _PROVIDERS = [
     badge: null,
     icon: <Mistral size={16} />,
     baseUrl: "https://api.mistral.ai/v1",
-    models: ["voxtral-mini-2602"],
+    models: ["voxtral-mini-2602", "voxtral-mini-transcribe-realtime-2602"],
     requirements: [{ kind: "requires_config", fields: ["api_key"] }],
   },
   {

crates/owhisper-client/src/adapter/elevenlabs/batch.rs

@@ -71,13 +71,7 @@ impl ElevenLabsAdapter {
             ))
         })?;
 
-        let default = crate::providers::Provider::ElevenLabs.default_batch_model();
-        let model = match params.model.as_deref() {
-            Some(m) if crate::providers::is_meta_model(m) => default,
-            Some("scribe_v2") => default,
-            Some(m) => m,
-            None => default,
-        };
+        let model = Self::resolve_batch_model(params.model.as_deref());
 
         let part = reqwest::multipart::Part::bytes(file_bytes).file_name(file_name);
         let mut form = reqwest::multipart::Form::new()
@@ -124,6 +118,16 @@ impl ElevenLabsAdapter {
         params.num_speakers.or(params.max_speakers)
     }
 
+    fn resolve_batch_model(model: Option<&str>) -> &str {
+        let default = crate::providers::Provider::ElevenLabs.default_batch_model();
+        match model {
+            Some(m) if crate::providers::is_meta_model(m) => default,
+            Some("scribe_v2" | "scribe_v2_realtime") => default,
+            Some(m) => m,
+            None => default,
+        }
+    }
+
     fn convert_to_batch_response(response: TranscriptResponse) -> BatchResponse {
         let words: Vec<BatchWord> = response
             .words
@@ -192,6 +196,14 @@ mod tests {
         );
     }
 
+    #[test]
+    fn batch_realtime_model_alias_uses_batch_model() {
+        assert_eq!(
+            ElevenLabsAdapter::resolve_batch_model(Some("scribe_v2_realtime")),
+            "scribe_v2"
+        );
+    }
+
     #[test]
     fn speaker_labeled_words_use_mixed_capture_channel() {
         let response = TranscriptResponse {

crates/owhisper-client/src/adapter/elevenlabs/live.rs

@@ -34,13 +34,7 @@ impl RealtimeSttAdapter for ElevenLabsAdapter {
                 query_pairs.append_pair(key, value);
             }
 
-            let default = crate::providers::Provider::ElevenLabs.default_live_model();
-            let model = match params.model.as_deref() {
-                Some(m) if crate::providers::is_meta_model(m) => default,
-                Some("scribe_v2") => default,
-                Some(m) => m,
-                None => default,
-            };
+            let model = Self::resolve_live_model(params.model.as_deref());
             query_pairs.append_pair("model_id", model);
 
             let audio_format = format!("pcm_{}", params.sample_rate);
@@ -201,6 +195,16 @@ enum ElevenLabsMessage {
 }
 
 impl ElevenLabsAdapter {
+    fn resolve_live_model(model: Option<&str>) -> &str {
+        let default = crate::providers::Provider::ElevenLabs.default_live_model();
+        match model {
+            Some(m) if crate::providers::is_meta_model(m) => default,
+            Some("scribe_v2") => default,
+            Some(m) => m,
+            None => default,
+        }
+    }
+
     fn build_response(
         text: &str,
         words: Vec<ElevenLabsWord>,

crates/owhisper-client/src/adapter/mistral/batch.rs

@@ -95,12 +95,7 @@ async fn do_transcribe_file(
         .mime_str(mime_type)
         .map_err(|e| Error::AudioProcessing(e.to_string()))?;
 
-    let default = Provider::Mistral.default_batch_model();
-    let model = match params.model.as_deref() {
-        Some(m) if is_meta_model(m) => default,
-        Some(m) => m,
-        None => default,
-    };
+    let model = resolve_batch_model(params.model.as_deref());
 
     let mut form = Form::new()
         .part("file", file_part)
@@ -149,6 +144,16 @@ fn strip_punctuation(s: &str) -> String {
         .to_string()
 }
 
+fn resolve_batch_model(model: Option<&str>) -> &str {
+    let default = Provider::Mistral.default_batch_model();
+    match model {
+        Some(m) if is_meta_model(m) => default,
+        Some("voxtral-mini-transcribe-realtime-2602") => default,
+        Some(m) => m,
+        None => default,
+    }
+}
+
 fn convert_response(response: MistralBatchResponse) -> BatchResponse {
     let (words, timing_source): (Vec<Word>, &str) = if !response.words.is_empty() {
         (
@@ -247,6 +252,14 @@ mod tests {
     use crate::adapter::BatchSttAdapter;
     use crate::http_client::create_client;
 
+    #[test]
+    fn batch_realtime_model_alias_uses_batch_model() {
+        assert_eq!(
+            resolve_batch_model(Some("voxtral-mini-transcribe-realtime-2602")),
+            "voxtral-mini-2602"
+        );
+    }
+
     #[test]
     fn convert_response_marks_segment_interpolated_words() {
         let response = convert_response(MistralBatchResponse {

crates/transcribe-proxy/src/relay/builder.rs

@@ -7,8 +7,8 @@ pub use tokio_tungstenite::tungstenite::ClientRequestBuilder;
 
 use super::handler::WebSocketProxy;
 use super::types::{
-    ClientMessageFilter, FirstMessageTransformer, InitialMessage, OnCloseCallback,
-    ResponseTransformer,
+    ClientBinaryMessageMapper, ClientMessageFilter, FirstMessageTransformer, InitialMessage,
+    OnCloseCallback, ResponseTransformer,
 };
 use crate::config::DEFAULT_CONNECT_TIMEOUT_MS;
 use crate::provider_selector::SelectedProvider;
@@ -38,6 +38,7 @@ pub struct WebSocketProxyBuilder<S = NoUpstream> {
     connect_timeout: Duration,
     on_close: Option<OnCloseCallback>,
     client_message_filter: Option<ClientMessageFilter>,
+    client_binary_message_mapper: Option<ClientBinaryMessageMapper>,
 }
 
 impl Default for WebSocketProxyBuilder<NoUpstream> {
@@ -51,6 +52,7 @@ impl Default for WebSocketProxyBuilder<NoUpstream> {
             connect_timeout: Duration::from_millis(DEFAULT_CONNECT_TIMEOUT_MS),
             on_close: None,
             client_message_filter: None,
+            client_binary_message_mapper: None,
         }
     }
 }
@@ -66,6 +68,7 @@ impl<S> WebSocketProxyBuilder<S> {
             connect_timeout: self.connect_timeout,
             on_close: self.on_close,
             client_message_filter: self.client_message_filter,
+            client_binary_message_mapper: self.client_binary_message_mapper,
         }
     }
 
@@ -79,6 +82,7 @@ impl<S> WebSocketProxyBuilder<S> {
         connect_timeout: Duration,
         on_close: Option<OnCloseCallback>,
         client_message_filter: Option<ClientMessageFilter>,
+        client_binary_message_mapper: Option<ClientBinaryMessageMapper>,
     ) -> WebSocketProxy {
         let control_message_types = if control_message_types.is_empty() {
             None
@@ -95,6 +99,7 @@ impl<S> WebSocketProxyBuilder<S> {
             connect_timeout,
             on_close,
             client_message_filter,
+            client_binary_message_mapper,
         )
     }
 
@@ -145,6 +150,11 @@ impl<S> WebSocketProxyBuilder<S> {
         self.client_message_filter = Some(filter);
         self
     }
+
+    pub fn client_binary_message_mapper(mut self, mapper: ClientBinaryMessageMapper) -> Self {
+        self.client_binary_message_mapper = Some(mapper);
+        self
+    }
 }
 
 impl WebSocketProxyBuilder<NoUpstream> {
@@ -209,6 +219,7 @@ impl WebSocketProxyBuilder<WithUrl> {
             self.connect_timeout,
             self.on_close,
             self.client_message_filter,
+            self.client_binary_message_mapper,
         ))
     }
 }

crates/transcribe-proxy/src/relay/channel_split/io.rs

@@ -3,8 +3,8 @@ use futures_util::{SinkExt, StreamExt};
 use tokio_tungstenite::tungstenite::Message as TungsteniteMessage;
 
 use super::super::types::{
-    ClientMessageFilter, ClientReceiver, ClientSender, DEFAULT_CLOSE_CODE, ShutdownSignal,
-    UpstreamReceiver, UpstreamSender, convert,
+    ClientBinaryMessage, ClientBinaryMessageMapper, ClientMessageFilter, ClientReceiver,
+    ClientSender, DEFAULT_CLOSE_CODE, ShutdownSignal, UpstreamReceiver, UpstreamSender, convert,
 };
 use super::coordinator::SplitEvent;
 use super::payload::RewrittenSplitResponse;
@@ -74,6 +74,7 @@ pub(super) async fn relay_client_to_upstreams(
     mut mic_tx: UpstreamSender,
     mut spk_tx: UpstreamSender,
     client_message_filter: Option<ClientMessageFilter>,
+    client_binary_message_mapper: Option<ClientBinaryMessageMapper>,
     shutdown_tx: tokio::sync::broadcast::Sender<ShutdownSignal>,
     event_tx: tokio::sync::mpsc::Sender<SplitEvent>,
 ) {
@@ -122,14 +123,30 @@ pub(super) async fn relay_client_to_upstreams(
                         }
 
                         let (mic, spk) = deinterleave(&bytes);
+                        let to_upstream = |data: Vec<u8>| {
+                            let mapped = match client_binary_message_mapper.as_ref() {
+                                Some(mapper) => mapper(data)?,
+                                None => ClientBinaryMessage::Binary(data),
+                            };
+
+                            Some(match mapped {
+                                ClientBinaryMessage::Text(text) => TungsteniteMessage::Text(text.into()),
+                                ClientBinaryMessage::Binary(data) => TungsteniteMessage::Binary(data.into()),
+                            })
+                        };
+
+                        let Some(mic_message) = to_upstream(mic) else {
+                            continue;
+                        };
+                        let Some(spk_message) = to_upstream(spk) else {
+                            continue;
+                        };
+
                         if mic_tx
-                            .send(TungsteniteMessage::Binary(mic.into()))
+                            .send(mic_message)
                             .await
                             .is_err()
-                            || spk_tx
-                                .send(TungsteniteMessage::Binary(spk.into()))
-                                .await
-                                .is_err()
+                            || spk_tx.send(spk_message).await.is_err()
                         {
                             let _ = event_tx
                                 .send(SplitEvent::Fatal(upstream_send_failed_signal()))