[AI] Support SpeechConfig in Text-To-Speech (#16226)

Co-authored-by: Ryan Wilson <wilsonryan@google.com>
Co-authored-by: Daymon <daymxn@google.com>
Co-authored-by: Nick Cooke <36927374+ncooke3@users.noreply.github.com>
Co-authored-by: Daymon <17409137+daymxn@users.noreply.github.com>
Co-authored-by: Nick Cooke <nickcooke@google.com>

Author: 4 people

.github/workflows/sdk.ai.yml

@@ -34,6 +34,7 @@ jobs:
     with:
       target: ${{  matrix.target }}
       setup_command: scripts/update_vertexai_responses.sh
+      env_vars: '{"FIREBASE_APP_CHECK_BRANCH": "main"}'
 
   testapp-integration:
     if: (github.repository == 'Firebase/firebase-ios-sdk' && github.event_name == 'schedule') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false)
@@ -69,6 +70,7 @@ jobs:
       TEST_RUNNER_VTXIntegrationImagen: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}
       FIREBASECI_USE_LATEST_GOOGLEAPPMEASUREMENT: 1
       secrets_passphrase: ${{ secrets.GHASecretsGPGPassphrase1 }}
+      FIREBASE_APP_CHECK_BRANCH: main
     steps:
     - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
     - uses: actions/cache/restore@9255dc7a253b0ccc959486e2bca901246202afeb # v5.0.1
@@ -108,6 +110,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     env:
       BRANCH_NAME: ${{ github.head_ref || github.ref_name || 'main' }}
+      FIREBASE_APP_CHECK_BRANCH: main
     steps:
     - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
     - name: Xcode

FirebaseAI/CHANGELOG.md

@@ -3,6 +3,8 @@
   simplify App Check setup. (#16185)
 - [fixed] Fixed a namespace collision with the new
   `FoundationModels.LanguageModelSession.Error` type introduced in Xcode 27 Beta. (#16252)
+- [feature] Added support for `SpeechConfig` in `GenerationConfig`, and `MultiSpeakerVoiceConfig`
+  in `SpeechConfig`. (#16226)
 
 # 12.14.0
 - [fixed] Fixed an issue in `GenerativeModelSession` where `String` generation

FirebaseAI/Sources/GenerationConfig.swift

@@ -61,6 +61,9 @@ public struct GenerationConfig: Sendable, Equatable {
   /// Configuration options for generating images.
   var imageConfig: ImageConfig?
 
+  /// Configuration for controlling the voice of the model during conversation.
+  var speechConfig: ProtoSpeechConfig?
+
   /// Creates a new `GenerationConfig` value.
   ///
   /// See the
@@ -164,13 +167,16 @@ public struct GenerationConfig: Sendable, Equatable {
   ///     > backwards-incompatible ways.
   ///   - thinkingConfig: Configuration for controlling the "thinking" behavior of compatible Gemini
   ///     models; see ``ThinkingConfig`` for more details.
+  ///   - speechConfig: Configuration for controlling the voice of the model during conversation;
+  ///     see ``SpeechConfig`` for more details.
   ///   - imageConfig: Configuration options for generating images.
   public init(temperature: Float? = nil, topP: Float? = nil, topK: Int? = nil,
               candidateCount: Int? = nil, maxOutputTokens: Int? = nil,
               presencePenalty: Float? = nil, frequencyPenalty: Float? = nil,
               stopSequences: [String]? = nil, responseMIMEType: String? = nil,
               responseSchema: Schema? = nil, responseModalities: [ResponseModality]? = nil,
-              thinkingConfig: ThinkingConfig? = nil, imageConfig: ImageConfig? = nil) {
+              thinkingConfig: ThinkingConfig? = nil, imageConfig: ImageConfig? = nil,
+              speechConfig: SpeechConfig? = nil) {
     // Explicit init because otherwise if we re-arrange the above variables it changes the API
     // surface.
     self.temperature = temperature
@@ -187,13 +193,14 @@ public struct GenerationConfig: Sendable, Equatable {
     self.responseModalities = responseModalities
     self.thinkingConfig = thinkingConfig
     self.imageConfig = imageConfig
+    self.speechConfig = speechConfig?.speechConfig
   }
 
   init(temperature: Float? = nil, topP: Float? = nil, topK: Int? = nil, candidateCount: Int? = nil,
        maxOutputTokens: Int? = nil, presencePenalty: Float? = nil, frequencyPenalty: Float? = nil,
        stopSequences: [String]? = nil, responseMIMEType: String, responseJSONSchema: JSONObject,
        responseModalities: [ResponseModality]? = nil, thinkingConfig: ThinkingConfig? = nil,
-       imageConfig: ImageConfig? = nil) {
+       imageConfig: ImageConfig? = nil, speechConfig: SpeechConfig? = nil) {
     self.temperature = temperature
     self.topP = topP
     self.topK = topK
@@ -207,6 +214,7 @@ public struct GenerationConfig: Sendable, Equatable {
     self.responseJSONSchema = responseJSONSchema
     self.responseModalities = responseModalities
     self.thinkingConfig = thinkingConfig
+    self.speechConfig = speechConfig?.speechConfig
     self.imageConfig = imageConfig
   }
 
@@ -246,6 +254,7 @@ public struct GenerationConfig: Sendable, Equatable {
     config.responseModalities = overrideConfig.responseModalities ?? config.responseModalities
     config.thinkingConfig = overrideConfig.thinkingConfig ?? config.thinkingConfig
     config.imageConfig = overrideConfig.imageConfig ?? config.imageConfig
+    config.speechConfig = overrideConfig.speechConfig ?? config.speechConfig
 
     // 5. Handle Schema mutual exclusivity with precedence for `responseJSONSchema`.
     if let responseJSONSchema = overrideConfig.responseJSONSchema {
@@ -278,5 +287,6 @@ extension GenerationConfig: Encodable {
     case responseModalities
     case thinkingConfig
     case imageConfig
+    case speechConfig
   }
 }

FirebaseAI/Sources/Types/Internal/Live/BidiGenerationConfig.swift

@@ -25,13 +25,13 @@ struct BidiGenerationConfig: Encodable, Sendable {
   let presencePenalty: Float?
   let frequencyPenalty: Float?
   let responseModalities: [ResponseModality]?
-  let speechConfig: BidiSpeechConfig?
+  let speechConfig: ProtoSpeechConfig?
 
   init(temperature: Float? = nil, topP: Float? = nil, topK: Int? = nil,
        candidateCount: Int? = nil, maxOutputTokens: Int? = nil,
        presencePenalty: Float? = nil, frequencyPenalty: Float? = nil,
        responseModalities: [ResponseModality]? = nil,
-       speechConfig: BidiSpeechConfig? = nil) {
+       speechConfig: ProtoSpeechConfig? = nil) {
     self.temperature = temperature
     self.topP = topP
     self.topK = topK

FirebaseAI/Sources/Types/Internal/Shared/ProtoMultiSpeakerVoiceConfig.swift

@@ -0,0 +1,23 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Foundation
+
+struct ProtoMultiSpeakerVoiceConfig: Encodable, Sendable, Equatable {
+  let speakerVoiceConfigs: [ProtoSpeakerVoiceConfig]
+
+  init(speakerVoiceConfigs: [ProtoSpeakerVoiceConfig]) {
+    self.speakerVoiceConfigs = speakerVoiceConfigs
+  }
+}

…pes/Internal/Live/BidiSpeechConfig.swift …nal/Shared/ProtoSpeakerVoiceConfig.swiftFirebaseAI/Sources/Types/Internal/Live/BidiSpeechConfig.swift renamed to FirebaseAI/Sources/Types/Internal/Shared/ProtoSpeakerVoiceConfig.swift FirebaseAI/Sources/Types/Internal/Live/BidiSpeechConfig.swift renamed to FirebaseAI/Sources/Types/Internal/Shared/ProtoSpeakerVoiceConfig.swift

@@ -1,4 +1,4 @@
-// Copyright 2025 Google LLC
+// Copyright 2026 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -14,17 +14,12 @@
 
 import Foundation
 
-/// Speech generation config.
-@available(watchOS, unavailable)
-struct BidiSpeechConfig: Encodable, Sendable {
-  /// The configuration for the speaker to use.
-  let voiceConfig: VoiceConfig
+struct ProtoSpeakerVoiceConfig: Encodable, Sendable, Equatable {
+  let speaker: String
+  let voiceConfig: ProtoVoiceConfig
 
-  /// Language code (ISO 639. e.g. en-US) for the speech synthesization.
-  let languageCode: String?
-
-  init(voiceConfig: VoiceConfig, languageCode: String?) {
+  init(speaker: String, voiceConfig: ProtoVoiceConfig) {
+    self.speaker = speaker
     self.voiceConfig = voiceConfig
-    self.languageCode = languageCode
   }
 }

FirebaseAI/Sources/Types/Internal/Shared/ProtoSpeechConfig.swift

@@ -0,0 +1,39 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Foundation
+
+/// Speech generation config.
+struct ProtoSpeechConfig: Encodable, Sendable, Equatable {
+  /// The configuration for the speaker to use.
+  let voiceConfig: ProtoVoiceConfig?
+
+  /// The configuration for the multi-speaker setup.
+  let multiSpeakerVoiceConfig: ProtoMultiSpeakerVoiceConfig?
+
+  /// Language code (BCP-47. e.g. en-US) for the speech synthesization.
+  let languageCode: String?
+
+  init(voiceConfig: ProtoVoiceConfig, languageCode: String?) {
+    self.voiceConfig = voiceConfig
+    multiSpeakerVoiceConfig = nil
+    self.languageCode = languageCode
+  }
+
+  init(multiSpeakerVoiceConfig: ProtoMultiSpeakerVoiceConfig, languageCode: String?) {
+    voiceConfig = nil
+    self.multiSpeakerVoiceConfig = multiSpeakerVoiceConfig
+    self.languageCode = languageCode
+  }
+}

…es/Types/Internal/Live/VoiceConfig.swift …s/Internal/Shared/ProtoVoiceConfig.swiftFirebaseAI/Sources/Types/Internal/Live/VoiceConfig.swift renamed to FirebaseAI/Sources/Types/Internal/Shared/ProtoVoiceConfig.swift FirebaseAI/Sources/Types/Internal/Live/VoiceConfig.swift renamed to FirebaseAI/Sources/Types/Internal/Shared/ProtoVoiceConfig.swift

@@ -15,21 +15,19 @@
 import Foundation
 
 /// Configuration for the speaker to use.
-@available(watchOS, unavailable)
-enum VoiceConfig {
+enum ProtoVoiceConfig: Sendable, Equatable {
   /// Configuration for the prebuilt voice to use.
-  case prebuiltVoiceConfig(PrebuiltVoiceConfig)
+  case prebuiltVoiceConfig(ProtoPrebuiltVoiceConfig)
 
   /// Configuration for the custom voice to use.
-  case customVoiceConfig(CustomVoiceConfig)
+  case customVoiceConfig(ProtoCustomVoiceConfig)
 }
 
 /// The configuration for the prebuilt speaker to use.
 ///
 /// Not just a string on the parent proto, because there'll likely be a lot
 /// more options here.
-@available(watchOS, unavailable)
-struct PrebuiltVoiceConfig: Encodable, Sendable {
+struct ProtoPrebuiltVoiceConfig: Encodable, Sendable, Equatable {
   /// The name of the preset voice to use.
   let voiceName: String
 
@@ -39,8 +37,7 @@ struct PrebuiltVoiceConfig: Encodable, Sendable {
 }
 
 /// The configuration for the custom voice to use.
-@available(watchOS, unavailable)
-struct CustomVoiceConfig: Encodable, Sendable {
+struct ProtoCustomVoiceConfig: Encodable, Sendable, Equatable {
   /// The sample of the custom voice, in pcm16 s16e format.
   let customVoiceSample: Data
 
@@ -51,8 +48,7 @@ struct CustomVoiceConfig: Encodable, Sendable {
 
 // MARK: - Encodable conformance
 
-@available(watchOS, unavailable)
-extension VoiceConfig: Encodable {
+extension ProtoVoiceConfig: Encodable {
   enum CodingKeys: CodingKey {
     case prebuiltVoiceConfig
     case customVoiceConfig

FirebaseAI/Sources/Types/Public/MultiSpeakerVoiceConfig.swift

@@ -0,0 +1,43 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Foundation
+
+/// Configuration for a multi-speaker audio generation setup.
+///
+/// Enables the model to generate audio containing multiple distinct speakers, alternating voices
+/// dynamically based on speaker labels in the prompt.
+///
+/// > Warning: Multi-speaker configurations are not currently supported by the Live API (e.g.,
+/// > `LiveGenerationConfig`).
+public struct MultiSpeakerVoiceConfig: Sendable {
+  let multiSpeakerVoiceConfig: ProtoMultiSpeakerVoiceConfig
+
+  init(_ multiSpeakerVoiceConfig: ProtoMultiSpeakerVoiceConfig) {
+    self.multiSpeakerVoiceConfig = multiSpeakerVoiceConfig
+  }
+
+  /// Creates a configuration for the multi-speaker setup.
+  ///
+  /// - Parameters:
+  ///   - speakerVoiceConfigs: A list of voice configurations for the participating speakers.
+  ///     Currently, the backend requires exactly **two** speaker voice configurations.
+  public init(speakerVoiceConfigs: [SpeakerVoiceConfig]) {
+    self.init(
+      ProtoMultiSpeakerVoiceConfig(
+        speakerVoiceConfigs: speakerVoiceConfigs.map(\.speakerVoiceConfig)
+      )
+    )
+  }
+}

FirebaseAI/Sources/Types/Public/SpeakerVoiceConfig.swift

@@ -0,0 +1,43 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Foundation
+
+/// Configures a speaker with a unique name/identifier and a specific voice.
+public struct SpeakerVoiceConfig: Sendable {
+  let speakerVoiceConfig: ProtoSpeakerVoiceConfig
+
+  init(_ speakerVoiceConfig: ProtoSpeakerVoiceConfig) {
+    self.speakerVoiceConfig = speakerVoiceConfig
+  }
+
+  /// Creates a configuration for a speaker using a voice name.
+  ///
+  /// - Parameters:
+  ///   - speaker: The unique name/identifier of the speaker (e.g., `"Alice"`).
+  ///   - voiceName: The name of the preset voice to assign to this speaker.
+  ///
+  /// Find the list of supported voices for:
+  /// - [Gemini Developer API](https://ai.google.dev/gemini-api/docs/speech-generation)
+  /// - [Vertex AI Gemini API](https://docs.cloud.google.com/text-to-speech/docs/gemini-tts)
+  // TODO(b/522397979): Update links to point to Firebase when they're live
+  public init(speaker: String, voiceName: String) {
+    self.init(
+      ProtoSpeakerVoiceConfig(
+        speaker: speaker,
+        voiceConfig: .prebuiltVoiceConfig(ProtoPrebuiltVoiceConfig(voiceName: voiceName))
+      )
+    )
+  }
+}