feat(elevenlabs): add STTv2 with streaming support for Scribe v2 (livekit#3909)

Co-authored-by: Claude <[email protected]>
Co-authored-by: Long Chen <[email protected]>

Author: 3 people

examples/other/realtime_scribe_v2.py

@@ -0,0 +1,42 @@
+import logging
+
+from dotenv import load_dotenv
+
+from livekit.agents import Agent, AgentSession, JobContext, JobProcess, WorkerOptions, cli
+from livekit.plugins import elevenlabs, openai, silero
+
+logger = logging.getLogger("realtime-scribe-v2")
+logger.setLevel(logging.INFO)
+
+load_dotenv()
+
+
+async def entrypoint(ctx: JobContext):
+    stt = elevenlabs.STTv2(
+        model_id="scribe_v2_realtime",
+        vad_silence_threshold_secs=0.5,
+        vad_threshold=0.5,
+        min_speech_duration_ms=100,
+        min_silence_duration_ms=300,
+    )
+
+    session = AgentSession(
+        allow_interruptions=True,
+        vad=ctx.proc.userdata["vad"],
+        stt=stt,
+        llm=openai.LLM(model="gpt-4.1-mini"),
+        tts=elevenlabs.TTS(model="eleven_turbo_v2_5"),
+    )
+    await session.start(
+        agent=Agent(instructions="You are a somewhat helpful assistant."), room=ctx.room
+    )
+
+    await session.say("Hello, how can I help you?")
+
+
+def prewarm(proc: JobProcess):
+    proc.userdata["vad"] = silero.VAD.load()
+
+
+if __name__ == "__main__":
+    cli.run_app(WorkerOptions(entrypoint_fnc=entrypoint, prewarm_fnc=prewarm))

livekit-plugins/livekit-plugins-elevenlabs/livekit/plugins/elevenlabs/__init__.py

@@ -17,18 +17,23 @@
 See https://docs.livekit.io/agents/integrations/tts/elevenlabs/ for more information.
 """
 
-from .models import TTSEncoding, TTSModels
+from .models import STTAudioFormat, STTModels, TTSEncoding, TTSModels
 from .stt import STT
+from .stt_v2 import SpeechStreamv2, STTv2
 from .tts import DEFAULT_VOICE_ID, TTS, Voice, VoiceSettings
 from .version import __version__
 
 __all__ = [
     "STT",
+    "STTv2",
+    "SpeechStreamv2",
     "TTS",
     "Voice",
     "VoiceSettings",
     "TTSEncoding",
     "TTSModels",
+    "STTModels",
+    "STTAudioFormat",
     "DEFAULT_VOICE_ID",
     "__version__",
 ]

livekit-plugins/livekit-plugins-elevenlabs/livekit/plugins/elevenlabs/models.py

@@ -20,3 +20,14 @@
     "mp3_44100_128",
     "mp3_44100_192",
 ]
+
+STTModels = Literal["scribe_v2_realtime",]
+
+STTAudioFormat = Literal[
+    "pcm_8000",
+    "pcm_16000",
+    "pcm_22050",
+    "pcm_24000",
+    "pcm_44100",
+    "pcm_48000",
+]