feat: wire up voice dictation in goose2 via ACP

Add voice dictation support to the goose2 Tauri app by exposing
transcription and config as ACP custom methods, then wiring the
frontend to use them.

Backend (crates/):
- Add DictationTranscribeRequest/Response and DictationConfigRequest/Response
  types to goose-sdk custom_requests.rs with model metadata fields
- Add #[custom_method] handlers in goose-acp server.rs for transcribe
  (OpenAI, Groq, ElevenLabs, Local) and config
- Register methods in acp-meta.json
- Forward local-inference feature from goose-cli to goose-acp

Tauri (ui/goose2/src-tauri/):
- Rewrite dictation.rs to use call_ext_method via ACP instead of
  importing goose crate directly
- Add generic CallExt command to ACP manager with method name
  normalization (strips leading _ to avoid double-prefix)
- Register get_dictation_config and transcribe_dictation commands

Frontend (ui/goose2/src/):
- Wire useDictationRecorder + useVoiceInputPreferences into ChatInput
- Replace placeholder mic button with working toggle (recording/
  transcribing states, auto-submit on keyword)
- Stop recording on manual send and on auto-submit keyword
- Show "Listening..."/"Transcribing..." placeholder in textarea
- Add Voice section to SettingsModal with VoiceInputSettings
- Add all voice i18n strings (en + es)
- Fix pre-existing type errors in dictationVad.ts and VoiceInputSettings

Known issue: Local Whisper reports configured: false despite model being
downloaded and config set. The is_downloaded() path check needs
investigation in a follow-up.

Co-Authored-By: Claude Opus 4.6 (1M context) <[email protected]>

Author: tulsi-builder

Cargo.lock

@@ -14,6 +14,7 @@ path = "src/bin/generate_acp_schema.rs"
 [features]
 default = ["code-mode", "rustls-tls"]
 code-mode = ["goose/code-mode"]
+local-inference = ["goose/local-inference"]
 rustls-tls = ["goose/rustls-tls", "goose-mcp/rustls-tls"]
 native-tls = ["goose/native-tls", "goose-mcp/native-tls"]
 
@@ -48,6 +49,7 @@ uuid = { workspace = true, features = ["v7"] }
 schemars = { workspace = true, features = ["derive"] }
 goose-acp-macros = { path = "../goose-acp-macros" }
 goose-sdk = { path = "../goose-sdk" }
+base64 = { workspace = true }
 
 [dev-dependencies]
 async-trait = { workspace = true }

crates/goose-acp/Cargo.toml

@@ -109,6 +109,16 @@
       "method": "_goose/session/unarchive",
       "requestType": "UnarchiveSessionRequest",
       "responseType": "EmptyResponse"
+    },
+    {
+      "method": "_goose/dictation/transcribe",
+      "requestType": "DictationTranscribeRequest",
+      "responseType": "DictationTranscribeResponse"
+    },
+    {
+      "method": "_goose/dictation/config",
+      "requestType": "DictationConfigRequest",
+      "responseType": "DictationConfigResponse"
     }
   ]
 }

crates/goose-acp/acp-meta.json

@@ -16,6 +16,13 @@ use goose::config::paths::Paths;
 use goose::config::permission::PermissionManager;
 use goose::config::{Config, GooseMode};
 use goose::conversation::message::{ActionRequiredData, Message, MessageContent};
+#[cfg(feature = "local-inference")]
+use goose::dictation::providers::transcribe_local;
+use goose::dictation::providers::{
+    all_providers, is_configured, transcribe_with_provider, DictationProvider,
+};
+#[cfg(feature = "local-inference")]
+use goose::dictation::whisper;
 use goose::mcp_utils::ToolResult;
 use goose::permission::permission_confirmation::PrincipalType;
 use goose::permission::{Permission, PermissionConfirmation};
@@ -68,6 +75,9 @@ pub type AcpProviderFactory = Arc<
 
 const DEFAULT_PROVIDER_ID: &str = "goose";
 const DEFAULT_PROVIDER_LABEL: &str = "Goose (Default)";
+const OPENAI_TRANSCRIPTION_MODEL: &str = "whisper-1";
+const GROQ_TRANSCRIPTION_MODEL: &str = "whisper-large-v3-turbo";
+const ELEVENLABS_TRANSCRIPTION_MODEL: &str = "scribe_v1";
 
 /// In-memory state for an active ACP session.
 ///
@@ -2651,6 +2661,197 @@ impl GooseAcpAgent {
             .map_err(|e| sacp::Error::internal_error().data(e.to_string()))?;
         Ok(EmptyResponse {})
     }
+
+    #[custom_method(DictationTranscribeRequest)]
+    async fn on_dictation_transcribe(
+        &self,
+        req: DictationTranscribeRequest,
+    ) -> Result<DictationTranscribeResponse, sacp::Error> {
+        use base64::{engine::general_purpose::STANDARD as BASE64, Engine};
+
+        let provider: DictationProvider = serde_json::from_value(serde_json::Value::String(
+            req.provider.clone(),
+        ))
+        .map_err(|_| {
+            sacp::Error::invalid_params().data(format!("Unknown provider: {}", req.provider))
+        })?;
+
+        let audio_bytes = BASE64
+            .decode(&req.audio)
+            .map_err(|_| sacp::Error::invalid_params().data("Invalid base64 audio data"))?;
+
+        if audio_bytes.len() > 50 * 1024 * 1024 {
+            return Err(sacp::Error::invalid_params().data("Audio too large (max 50MB)"));
+        }
+
+        let extension = match req.mime_type.as_str() {
+            "audio/webm" | "audio/webm;codecs=opus" => "webm",
+            "audio/mp4" => "mp4",
+            "audio/mpeg" | "audio/mpga" => "mp3",
+            "audio/m4a" => "m4a",
+            "audio/wav" | "audio/x-wav" => "wav",
+            other => {
+                return Err(
+                    sacp::Error::invalid_params().data(format!("Unsupported format: {other}"))
+                )
+            }
+        };
+
+        let text = match provider {
+            DictationProvider::OpenAI => {
+                transcribe_with_provider(
+                    DictationProvider::OpenAI,
+                    "model".to_string(),
+                    "whisper-1".to_string(),
+                    audio_bytes,
+                    extension,
+                    &req.mime_type,
+                )
+                .await
+            }
+            DictationProvider::Groq => {
+                transcribe_with_provider(
+                    DictationProvider::Groq,
+                    "model".to_string(),
+                    "whisper-large-v3-turbo".to_string(),
+                    audio_bytes,
+                    extension,
+                    &req.mime_type,
+                )
+                .await
+            }
+            DictationProvider::ElevenLabs => {
+                transcribe_with_provider(
+                    DictationProvider::ElevenLabs,
+                    "model_id".to_string(),
+                    "scribe_v1".to_string(),
+                    audio_bytes,
+                    extension,
+                    &req.mime_type,
+                )
+                .await
+            }
+            #[cfg(feature = "local-inference")]
+            DictationProvider::Local => transcribe_local(audio_bytes).await,
+            #[cfg(not(feature = "local-inference"))]
+            DictationProvider::Local => {
+                return Err(sacp::Error::invalid_params()
+                    .data("Local inference is not available in this build"));
+            }
+        }
+        .map_err(|e| sacp::Error::internal_error().data(e.to_string()))?;
+
+        Ok(DictationTranscribeResponse { text })
+    }
+
+    #[custom_method(DictationConfigRequest)]
+    async fn on_dictation_config(
+        &self,
+        _req: DictationConfigRequest,
+    ) -> Result<DictationConfigResponse, sacp::Error> {
+        let config = goose::config::Config::global();
+        let mut providers = std::collections::HashMap::new();
+
+        for def in all_providers() {
+            let provider = def.provider;
+            let host = if let Some(host_key) = def.host_key {
+                config
+                    .get(host_key, false)
+                    .ok()
+                    .and_then(|v| v.as_str().map(|s| s.to_string()))
+            } else {
+                None
+            };
+
+            let provider_key = serde_json::to_value(provider)
+                .ok()
+                .and_then(|v| v.as_str().map(|s| s.to_string()))
+                .unwrap_or_else(|| format!("{:?}", provider).to_lowercase());
+            providers.insert(
+                provider_key,
+                DictationProviderStatusEntry {
+                    configured: is_configured(provider),
+                    host,
+                    description: def.description.to_string(),
+                    uses_provider_config: def.uses_provider_config,
+                    settings_path: def.settings_path.map(|s| s.to_string()),
+                    config_key: if !def.uses_provider_config {
+                        Some(def.config_key.to_string())
+                    } else {
+                        None
+                    },
+                    model_config_key: dictation_model_config_key(provider),
+                    default_model: dictation_default_model(provider),
+                    selected_model: dictation_selected_model(&config, provider),
+                    available_models: dictation_available_models(provider),
+                },
+            );
+        }
+
+        Ok(DictationConfigResponse { providers })
+    }
+}
+
+fn dictation_model_config_key(provider: DictationProvider) -> Option<String> {
+    #[cfg(feature = "local-inference")]
+    if provider == DictationProvider::Local {
+        return Some(whisper::LOCAL_WHISPER_MODEL_CONFIG_KEY.to_string());
+    }
+
+    None
+}
+
+fn dictation_default_model(provider: DictationProvider) -> Option<String> {
+    match provider {
+        DictationProvider::OpenAI => Some(OPENAI_TRANSCRIPTION_MODEL.to_string()),
+        DictationProvider::Groq => Some(GROQ_TRANSCRIPTION_MODEL.to_string()),
+        DictationProvider::ElevenLabs => Some(ELEVENLABS_TRANSCRIPTION_MODEL.to_string()),
+        #[cfg(feature = "local-inference")]
+        DictationProvider::Local => Some(whisper::recommend_model().to_string()),
+    }
+}
+
+fn dictation_selected_model(config: &Config, provider: DictationProvider) -> Option<String> {
+    #[cfg(feature = "local-inference")]
+    if provider == DictationProvider::Local {
+        return config
+            .get(whisper::LOCAL_WHISPER_MODEL_CONFIG_KEY, false)
+            .ok()
+            .and_then(|value| value.as_str().map(str::to_owned))
+            .filter(|model_id| whisper::get_model(model_id).is_some())
+            .or_else(|| dictation_default_model(provider));
+    }
+
+    dictation_default_model(provider)
+}
+
+fn dictation_available_models(provider: DictationProvider) -> Vec<DictationModelOption> {
+    match provider {
+        DictationProvider::OpenAI => vec![DictationModelOption {
+            id: OPENAI_TRANSCRIPTION_MODEL.to_string(),
+            label: "Whisper-1".to_string(),
+            description: "OpenAI's hosted Whisper transcription model.".to_string(),
+        }],
+        DictationProvider::Groq => vec![DictationModelOption {
+            id: GROQ_TRANSCRIPTION_MODEL.to_string(),
+            label: "Whisper Large V3 Turbo".to_string(),
+            description: "Groq's fast hosted Whisper transcription model.".to_string(),
+        }],
+        DictationProvider::ElevenLabs => vec![DictationModelOption {
+            id: ELEVENLABS_TRANSCRIPTION_MODEL.to_string(),
+            label: "Scribe v1".to_string(),
+            description: "ElevenLabs' hosted speech-to-text model.".to_string(),
+        }],
+        #[cfg(feature = "local-inference")]
+        DictationProvider::Local => whisper::available_models()
+            .iter()
+            .map(|model| DictationModelOption {
+                id: model.id.to_string(),
+                label: model.id.to_string(),
+                description: model.description.to_string(),
+            })
+            .collect(),
+    }
 }
 
 pub struct GooseAcpHandler {

crates/goose-acp/src/server.rs

@@ -71,7 +71,7 @@ winapi = { workspace = true }
 [features]
 default = ["code-mode", "local-inference", "aws-providers", "telemetry", "otel", "rustls-tls"]
 code-mode = ["goose/code-mode", "goose-acp/code-mode"]
-local-inference = ["goose/local-inference"]
+local-inference = ["goose/local-inference", "goose-acp/local-inference"]
 aws-providers = ["goose/aws-providers"]
 cuda = ["goose/cuda", "local-inference"]
 telemetry = ["goose/telemetry"]

crates/goose-cli/Cargo.toml

@@ -330,6 +330,66 @@ pub struct ProviderConfigKey {
     pub primary: bool,
 }
 
+/// Transcribe audio via a dictation provider.
+#[derive(Debug, Default, Clone, Serialize, Deserialize, JsonSchema, JsonRpcRequest)]
+#[request(method = "_goose/dictation/transcribe", response = DictationTranscribeResponse)]
+#[serde(rename_all = "camelCase")]
+pub struct DictationTranscribeRequest {
+    /// Base64-encoded audio data
+    pub audio: String,
+    /// MIME type (e.g. "audio/wav", "audio/webm")
+    pub mime_type: String,
+    /// Provider to use: "openai", "groq", "elevenlabs", or "local"
+    pub provider: String,
+}
+
+/// Transcription result.
+#[derive(Debug, Default, Clone, Serialize, Deserialize, JsonSchema, JsonRpcResponse)]
+pub struct DictationTranscribeResponse {
+    pub text: String,
+}
+
+/// Get the configuration status of all dictation providers.
+#[derive(Debug, Default, Clone, Serialize, Deserialize, JsonSchema, JsonRpcRequest)]
+#[request(method = "_goose/dictation/config", response = DictationConfigResponse)]
+pub struct DictationConfigRequest {}
+
+#[derive(Debug, Default, Clone, Serialize, Deserialize, JsonSchema)]
+pub struct DictationModelOption {
+    pub id: String,
+    pub label: String,
+    pub description: String,
+}
+
+/// Per-provider configuration status.
+#[derive(Debug, Default, Clone, Serialize, Deserialize, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct DictationProviderStatusEntry {
+    pub configured: bool,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub host: Option<String>,
+    pub description: String,
+    pub uses_provider_config: bool,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub settings_path: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub config_key: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub model_config_key: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub default_model: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub selected_model: Option<String>,
+    #[serde(default)]
+    pub available_models: Vec<DictationModelOption>,
+}
+
+/// Dictation config response — map of provider name to status.
+#[derive(Debug, Default, Clone, Serialize, Deserialize, JsonSchema, JsonRpcResponse)]
+pub struct DictationConfigResponse {
+    pub providers: HashMap<String, DictationProviderStatusEntry>,
+}
+
 /// Empty success response for operations that return no data.
 #[derive(Debug, Default, Clone, Serialize, Deserialize, JsonSchema, JsonRpcResponse)]
 pub struct EmptyResponse {}