Merge pull request #340 from mucsi96/claude/loving-mayer-r6nz60

Add Google Gemini TTS as switchable audio generation model

Author: mucsi96

AGENTS.md

@@ -59,7 +59,7 @@ This is a language learning application that uses spaced repetition to help user
 
 ### Key Technologies
 - **Spaced Repetition**: Uses FSRS (Free Spaced Repetition Scheduler) algorithm via ts-fsrs library
-- **AI Integration**: OpenAI GPT-4.1 for translations, ElevenLabs Voices for audio, Google Gemini API for example images
+- **AI Integration**: OpenAI GPT-4.1 for translations, ElevenLabs Voices and Google Gemini TTS for audio, Google Gemini API for example images
 - **Cloud Services**: Local file system for PDFs/assets storage, Azure AD for authentication
 - **PDF Processing**: Apache PDFBox for text extraction and document processing
 

client/src/app/environment/environment.config.ts

@@ -1,6 +1,6 @@
 import { InjectionToken } from '@angular/core';
 
-export type ModelProvider = 'openai' | 'anthropic' | 'google';
+export type ModelProvider = 'openai' | 'anthropic' | 'google' | 'elevenlabs';
 
 export interface ChatModelInfo {
   modelName: string;
@@ -17,13 +17,15 @@ export interface AudioModel {
   id: string;
   displayName: string;
   isDefault: boolean;
+  provider: ModelProvider;
 }
 
 export interface Voice {
   id: string;
   displayName: string;
   languages: { name: string }[];
   category: 'premade' | 'cloned' | 'generated' | 'professional' | null;
+  provider: ModelProvider;
 }
 
 export interface SupportedLanguage {

client/src/app/voice-config/add-voice-dialog/add-voice-dialog.component.html

@@ -43,7 +43,7 @@ <h2 mat-dialog-title>Add Voice Configuration</h2>
       <mat-select
         [formField]="voiceForm.model"
       >
-        @for (model of models; track model.id) {
+        @for (model of availableModels(); track model.id) {
           <mat-option [value]="model.id">
             {{ model.displayName }}
             @if (model.isDefault) {

client/src/app/voice-config/add-voice-dialog/add-voice-dialog.component.ts

@@ -53,11 +53,18 @@ export class AddVoiceDialogComponent {
   });
   readonly voiceForm = form(this.formModel);
 
-  readonly models = this.data.audioModels;
+  readonly availableModels = computed(() => {
+    const voice = this.formModel().voice;
+    if (!voice) return this.data.audioModels;
+
+    return this.data.audioModels.filter(
+      (model) => model.provider === voice.provider
+    );
+  });
 
-  private getDefaultModelId(): string {
-    const defaultModel = this.data.audioModels.find((m) => m.isDefault);
-    return defaultModel?.id ?? this.data.audioModels[0]?.id ?? '';
+  private getDefaultModelId(models = this.data.audioModels): string {
+    const defaultModel = models.find((m) => m.isDefault);
+    return defaultModel?.id ?? models[0]?.id ?? '';
   }
 
   readonly filteredVoices = computed(() => this.data.availableVoices);
@@ -81,6 +88,7 @@ export class AddVoiceDialogComponent {
     this.formModel.update((m) => ({
       ...m,
       language: langs.length === 1 ? langs[0] : '',
+      model: this.getDefaultModelId(this.availableModels()),
       displayName: m.voice?.displayName ?? '',
     }));
   }

mock_google_ai_server/src/audioGeneration.ts

@@ -0,0 +1,42 @@
+const SAMPLE_RATE = 24000;
+const BEEP_DURATION_SECONDS = 0.3;
+const BEEP_AMPLITUDE = 12000;
+
+// Raw 16-bit PCM mono sine beeps, matching the format Gemini TTS returns
+const generatePcmBeep = (frequency: number): string => {
+  const sampleCount = Math.floor(SAMPLE_RATE * BEEP_DURATION_SECONDS);
+  const samples = Int16Array.from({ length: sampleCount }, (_, i) =>
+    Math.round(Math.sin((2 * Math.PI * frequency * i) / SAMPLE_RATE) * BEEP_AMPLITUDE)
+  );
+  return Buffer.from(samples.buffer).toString('base64');
+};
+
+export const AUDIO_SAMPLES = {
+  german: generatePcmBeep(440),
+  hungarian: generatePcmBeep(660),
+};
+
+export class AudioGenerationHandler {
+  private audioCallCounter = 0;
+
+  reset(): void {
+    this.audioCallCounter = 0;
+  }
+
+  generateAudio(prompt: string, voiceName?: string): string {
+    this.audioCallCounter++;
+
+    const isHungarian = /Hungarian/i.test(prompt);
+    const audioBase64 = isHungarian ? AUDIO_SAMPLES.hungarian : AUDIO_SAMPLES.german;
+
+    console.log(
+      `Generated TTS audio for prompt: "${prompt}" (language: ${isHungarian ? 'hu' : 'de'}, voice: ${voiceName ?? 'unknown'})`
+    );
+
+    return audioBase64;
+  }
+
+  getCallCount(): number {
+    return this.audioCallCounter;
+  }
+}

mock_google_ai_server/src/index.ts

@@ -1,10 +1,12 @@
 import express from 'express';
 import { ImageGenerationHandler } from './imageGeneration';
 import { ChatHandler } from './chatHandler';
+import { AudioGenerationHandler } from './audioGeneration';
 
 const app = express();
 const imageHandler = new ImageGenerationHandler();
 const chatHandler = new ChatHandler();
+const audioHandler = new AudioGenerationHandler();
 
 app.use(express.json({ limit: '25mb' }));
 
@@ -20,7 +22,12 @@ app.use((req, res, next) => {
 app.post('/reset', (req, res) => {
   imageHandler.reset();
   chatHandler.reset();
-  res.status(200).json({ status: 'ok', message: 'Image counter reset to 0' });
+  audioHandler.reset();
+  res.status(200).json({ status: 'ok', message: 'Mock state reset' });
+});
+
+app.get('/stats', (req, res) => {
+  res.status(200).json({ audioCallCount: audioHandler.getCallCount() });
 });
 
 app.post('/configure', (req, res) => {
@@ -60,6 +67,38 @@ app.post(
   }
 );
 
+app.post(
+  '/v1beta/models/gemini-3.1-flash-tts-preview:generateContent',
+  (req, res) => {
+    try {
+      const prompt = req.body.contents[0].parts[0].text;
+      const voiceName =
+        req.body.generationConfig?.speechConfig?.voiceConfig
+          ?.prebuiltVoiceConfig?.voiceName;
+      const audio = audioHandler.generateAudio(prompt, voiceName);
+      res.status(200).json({
+        candidates: [
+          {
+            content: {
+              parts: [
+                {
+                  inlineData: {
+                    mimeType: 'audio/L16;codec=pcm;rate=24000',
+                    data: audio,
+                  },
+                },
+              ],
+            },
+          },
+        ],
+      });
+    } catch (error) {
+      console.error('Audio generation error:', error);
+      res.status(500).json({ error: { message: 'Audio generation failed' } });
+    }
+  }
+);
+
 app.post(/\/v1beta\/models\/([^/]+):generateContent/, async (req, res) => {
   try {
     const model = req.params[0];

server/src/main/java/io/github/mucsi96/learnlanguage/config/ModelPricingConfig.java

@@ -48,7 +48,9 @@ public record AudioModelPricing(BigDecimal perThousandCharacters) {}
     private static final Map<String, AudioModelPricing> AUDIO_MODEL_PRICING = Map.of(
         // ElevenLabs (approximately $0.20 per 1000 characters)
         "eleven_turbo_v2_5", new AudioModelPricing(new BigDecimal("0.20")),
-        "eleven_v3", new AudioModelPricing(new BigDecimal("0.20"))
+        "eleven_v3", new AudioModelPricing(new BigDecimal("0.20")),
+        // Gemini TTS is token-priced; approximated per 1000 characters
+        "gemini-3.1-flash-tts-preview", new AudioModelPricing(new BigDecimal("0.02"))
     );
 
     public ChatModelPricing getChatModelPricing(String modelName) {

server/src/main/java/io/github/mucsi96/learnlanguage/controller/EnvironmentController.java

@@ -3,6 +3,7 @@
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Stream;
 
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.web.bind.annotation.GetMapping;
@@ -20,6 +21,7 @@
 import io.github.mucsi96.learnlanguage.service.AudioSettingService;
 import io.github.mucsi96.learnlanguage.service.ChatModelSettingService;
 import io.github.mucsi96.learnlanguage.service.ElevenLabsAudioService;
+import io.github.mucsi96.learnlanguage.service.GeminiAudioService;
 import io.github.mucsi96.learnlanguage.service.ImageModelSettingService;
 import io.github.mucsi96.learnlanguage.service.ImageSettingService;
 import io.github.mucsi96.learnlanguage.service.RateLimitSettingService;
@@ -30,6 +32,7 @@
 public class EnvironmentController {
   private final AudioService audioService;
   private final ElevenLabsAudioService elevenLabsAudioService;
+  private final GeminiAudioService geminiAudioService;
   private final ChatModelSettingService chatModelSettingService;
   private final ImageModelSettingService imageModelSettingService;
   private final ImageSettingService imageSettingService;
@@ -101,7 +104,9 @@ public ConfigResponse getConfig() {
             .toList(),
         imageModelSettingService.getImageModelsWithSettings(),
         audioService.getAvailableModels(),
-        elevenLabsAudioService.getVoices(),
+        Stream.concat(
+            elevenLabsAudioService.getVoices().stream(),
+            geminiAudioService.getVoices().stream()).toList(),
         SUPPORTED_LANGUAGES,
         enabledModelsByOperation,
         primaryModelByOperation,

server/src/main/java/io/github/mucsi96/learnlanguage/model/AudioModelResponse.java

@@ -13,4 +13,5 @@ public class AudioModelResponse {
     private String id;
     private String displayName;
     private boolean isDefault;
+    private ModelProvider provider;
 }

server/src/main/java/io/github/mucsi96/learnlanguage/model/ModelProvider.java

@@ -10,7 +10,8 @@
 public enum ModelProvider {
     OPENAI("openai"),
     ANTHROPIC("anthropic"),
-    GOOGLE("google");
+    GOOGLE("google"),
+    ELEVENLABS("elevenlabs");
 
     private final String code;