Use faster-whisper for transcription and update transcription dependencies for Python 3.11 compatibility

README.md

@@ -14,7 +14,7 @@ Manim Voiceover is a [Manim](https://manim.community) plugin for all things voic
 - Add voiceovers to Manim videos *directly in Python* without having to use a video editor.
 - Record voiceovers with your microphone during rendering with a simple command line interface.
 - Develop animations with auto-generated AI voices from various free and proprietary services.
-- Per-word timing of animations, i.e. trigger animations at specific words in the voiceover, even for the recordings. This works thanks to [OpenAI Whisper](https://github.com/openai/whisper).
+- Per-word timing of animations, i.e. trigger animations at specific words in the voiceover, even for the recordings. This works thanks to [faster-whisper](https://github.com/SYSTRAN/faster-whisper).
 
 Here is a demo:
 

docs/source/index.rst

@@ -11,7 +11,7 @@ Manim Voiceover
 - Add voiceovers to Manim videos *directly in Python* without having to use a video editor.
 - Record voiceovers with your microphone during rendering with a simple command line interface (see :py:class:`~manim_voiceover.services.recorder.RecorderService`).
 - Develop animations with auto-generated AI voices from various free and proprietary services.
-- Per-word timing of animations, i.e. trigger animations at specific words in the voiceover, even for the recordings. This works thanks to `OpenAI Whisper <https://github.com/openai/whisper>`__.
+- Per-word timing of animations, i.e. trigger animations at specific words in the voiceover, even for the recordings. This works thanks to `faster-whisper <https://github.com/SYSTRAN/faster-whisper>`__.
 
 A demo:
 

manim_voiceover/__init__.py

@@ -1,6 +1,7 @@
 from manim_voiceover.tracker import VoiceoverTracker
 from manim_voiceover.voiceover_scene import VoiceoverScene
+from manim_voiceover.voiceover_extractor import VoiceoverExtractor
 
-import pkg_resources
+from importlib.metadata import version
 
-__version__: str = pkg_resources.get_distribution(__name__).version
+__version__: str = version(__name__)

manim_voiceover/helper.py

@@ -1,3 +1,4 @@
+import fcntl
 import importlib
 import json
 import re
@@ -101,22 +102,31 @@ def trim_silence(
 
 
 def append_to_json_file(json_file: str, data: dict):
-    """Append data to json file"""
-    if not os.path.exists(json_file):
-        with open(json_file, "w") as f:
-            json.dump([data], f, indent=2)
-        return
-
-    with open(json_file, "r") as f:
-        json_data = json.load(f)
-
-    if not isinstance(json_data, list):
-        raise ValueError("JSON file should be a list")
+    """Append data to JSON file.
+    Uses file locking to prevent corruption when multiple processes
+    write to the same cache file concurrently.
+    """
 
-    json_data.append(data)
-    with open(json_file, "w") as f:
-        json.dump(json_data, f, indent=2)
-    return
+    lock_path = str(json_file) + ".lock"
+    with open(lock_path, "w") as lock_f:
+        fcntl.flock(lock_f, fcntl.LOCK_EX)
+        try:
+            if not os.path.exists(json_file):
+                with open(json_file, "w") as f:
+                    json.dump([data], f, indent=2)
+                return
+
+            with open(json_file, "r") as f:
+                json_data = json.load(f)
+
+            if not isinstance(json_data, list):
+                raise ValueError("JSON file should be a list")
+
+            json_data.append(data)
+            with open(json_file, "w") as f:
+                json.dump(json_data, f, indent=2)
+        finally:
+            fcntl.flock(lock_f, fcntl.LOCK_UN)
 
 
 def prompt_ask_missing_package(target_module: str, package_name: str):

manim_voiceover/modify_audio.py

@@ -11,9 +11,10 @@ def adjust_speed(input_path: str, output_path: str, tempo: float) -> None:
         path_, ext = os.path.splitext(input_path)
         output_path = path_ + str(uuid.uuid1()) + ext
 
-    tfm = sox.Transformer()
-    tfm.tempo(tempo)
-    tfm.build(input_filepath=input_path, output_filepath=output_path)
+    # AudioStretchy ratio is inverted: ratio > 1.0 = slower, < 1.0 = faster.
+    # Our tempo convention: tempo > 1.0 = faster. So ratio = 1/tempo.
+    stretch_audio(input_path, output_path, ratio=1.0 / tempo)
+
     if same_destination:
         os.rename(output_path, input_path)
 

manim_voiceover/services/base.py

@@ -1,8 +1,8 @@
+import fcntl
 from abc import ABC, abstractmethod
 import typing as t
 import os
 import json
-import sys
 import hashlib
 from pathlib import Path
 from manim import config, logger
@@ -21,25 +21,31 @@
 
 
 def timestamps_to_word_boundaries(segments):
+    """Convert faster-whisper segments to word boundaries format.
+
+    Args:
+        segments: Iterator of faster-whisper Segment objects with word timestamps.
+
+    Returns:
+        list: List of word boundary dictionaries.
+    """
     word_boundaries = []
     current_text_offset = 0
     for segment in segments:
-        for dict_ in segment["words"]:
-            word = dict_["word"]
+        if segment.words is None:
+            continue
+        for word_info in segment.words:
+            word = word_info.word
             word_boundaries.append(
                 {
-                    "audio_offset": int(dict_["start"] * AUDIO_OFFSET_RESOLUTION),
-                    # "duration_milliseconds": 0,
+                    "audio_offset": int(word_info.start * AUDIO_OFFSET_RESOLUTION),
                     "text_offset": current_text_offset,
                     "word_length": len(word),
                     "text": word,
                     "boundary_type": "Word",
                 }
             )
             current_text_offset += len(word)
-            # If word is not punctuation, add a space
-            # if word not in [".", ",", "!", "?", ";", ":", "(", ")"]:
-            # current_text_offset += 1
 
     return word_boundaries
 
@@ -50,9 +56,10 @@ class SpeechService(ABC):
     def __init__(
         self,
         global_speed: float = 1.00,
-        cache_dir: t.Optional[str] = None,
+        cache_dir: t.Optional[t.Union[str, Path]] = None,
         transcription_model: t.Optional[str] = None,
         transcription_kwargs: dict = {},
+        transcription_model_kwargs: dict = {},
         **kwargs,
     ):
         """
@@ -62,24 +69,26 @@ def __init__(
             cache_dir (str, optional): The directory to save the audio
                 files to. Defaults to ``voiceovers/``.
             transcription_model (str, optional): The
-                `OpenAI Whisper model <https://github.com/openai/whisper#available-models-and-languages>`_
+                `Whisper model <https://github.com/SYSTRAN/faster-whisper#available-models>`_
                 to use for transcription. Defaults to None.
             transcription_kwargs (dict, optional): Keyword arguments to
                 pass to the transcribe() function. Defaults to {}.
+            transcription_model_kwargs (dict, optional): Keyword arguments to
+                pass to the WhisperModel constructor (e.g., compute_type, device).
+                Defaults to {}.
         """
         self.global_speed = global_speed
 
         if cache_dir is not None:
-            self.cache_dir = cache_dir
+            self.cache_dir = Path(cache_dir)
         else:
             self.cache_dir = Path(config.media_dir) / DEFAULT_VOICEOVER_CACHE_DIR
 
-        if not os.path.exists(self.cache_dir):
-            os.makedirs(self.cache_dir)
+        self.cache_dir.mkdir(parents=True, exist_ok=True)
 
         self.transcription_model = None
         self._whisper_model = None
-        self.set_transcription(model=transcription_model, kwargs=transcription_kwargs)
+        self.set_transcription(model=transcription_model, kwargs=transcription_kwargs, model_kwargs=transcription_model_kwargs)
 
         self.additional_kwargs = kwargs
 
@@ -88,19 +97,27 @@ def _wrap_generate_from_text(self, text: str, path: str = None, **kwargs) -> dic
         text = " ".join(text.split())
 
         dict_ = self.generate_from_text(text, cache_dir=None, path=path, **kwargs)
+
+        # If the result was already fully cached (including final_audio), return it
+        if "final_audio" in dict_ and (Path(self.cache_dir) / dict_["final_audio"]).exists():
+            return dict_
+
         original_audio = dict_["original_audio"]
 
         # Check whether word boundaries exist and if not run stt
         if "word_boundaries" not in dict_ and self._whisper_model is not None:
-            transcription_result = self._whisper_model.transcribe(
-                str(Path(self.cache_dir) / original_audio), **self.transcription_kwargs
-            )
-            logger.info("Transcription: " + transcription_result.text)
-            word_boundaries = timestamps_to_word_boundaries(
-                transcription_result.segments_to_dicts()
+            segments, info = self._whisper_model.transcribe(
+                str(Path(self.cache_dir) / original_audio),
+                word_timestamps=True,
+                **self.transcription_kwargs
             )
+            # Consume the generator to get all segments
+            segments_list = list(segments)
+            transcribed_text = "".join(segment.text for segment in segments_list)
+            logger.info("Transcription: " + transcribed_text)
+            word_boundaries = timestamps_to_word_boundaries(segments_list)
             dict_["word_boundaries"] = word_boundaries
-            dict_["transcribed_text"] = transcription_result.text
+            dict_["transcribed_text"] = transcribed_text
 
         # Audio callback
         self.audio_callback(original_audio, dict_, **kwargs)
@@ -123,39 +140,50 @@ def _wrap_generate_from_text(self, text: str, path: str = None, **kwargs) -> dic
         else:
             dict_["final_audio"] = dict_["original_audio"]
 
+        # Cache the result to a JSON file defined by Manim's cache directory
         append_to_json_file(
             Path(self.cache_dir) / DEFAULT_VOICEOVER_CACHE_JSON_FILENAME, dict_
         )
         return dict_
 
-    def set_transcription(self, model: str = None, kwargs: dict = {}):
+    def set_transcription(self, model: str = None, kwargs: dict = {}, model_kwargs: dict = {}):
         """Set the transcription model and keyword arguments to be passed
         to the transcribe() function.
 
         Args:
             model (str, optional): The Whisper model to use for transcription. Defaults to None.
             kwargs (dict, optional): Keyword arguments to pass to the transcribe() function. Defaults to {}.
+            model_kwargs (dict, optional): Keyword arguments to pass to the WhisperModel constructor
+                (e.g., compute_type, device). Defaults to {}.
         """
         if model != self.transcription_model:
             if model is not None:
                 try:
-                    import whisper as __tmp
-                    import stable_whisper as whisper
+                    from faster_whisper import WhisperModel
                 except ImportError:
                     logger.error(
                         'Missing packages. Run `pip install "manim-voiceover[transcribe]"` to be able to transcribe voiceovers.'
                     )
 
                 prompt_ask_missing_extras(
-                    ["whisper", "stable_whisper"],
+                    ["faster_whisper"],
                     "transcribe",
                     "SpeechService.set_transcription()",
                 )
-                self._whisper_model = whisper.load_model(model)
+
+                # Suppress verbose logging from huggingface_hub, httpx, faster_whisper, and ctranslate2
+                import logging
+                logging.getLogger("httpx").setLevel(logging.WARNING)
+                logging.getLogger("huggingface_hub").setLevel(logging.WARNING)
+                logging.getLogger("faster_whisper").setLevel(logging.WARNING)
+                logging.getLogger("ctranslate2").setLevel(logging.ERROR)
+
+                self._whisper_model = WhisperModel(model, **model_kwargs)
             else:
                 self._whisper_model = None
 
         self.transcription_kwargs = kwargs
+        self.transcription_model = model
 
     def get_audio_basename(self, data: dict) -> str:
         dumped_data = json.dumps(data)
@@ -186,10 +214,17 @@ def generate_from_text(
     def get_cached_result(self, input_data, cache_dir):
         json_path = os.path.join(cache_dir / DEFAULT_VOICEOVER_CACHE_JSON_FILENAME)
         if os.path.exists(json_path):
-            json_data = json.load(open(json_path, "r"))
-            for entry in json_data:
-                if entry["input_data"] == input_data:
-                    return entry
+            lock_path = str(json_path) + ".lock"
+            with open(lock_path, "w") as lock_f:
+                fcntl.flock(lock_f, fcntl.LOCK_SH)
+                try:
+                    with open(json_path, "r") as f:
+                        json_data = json.load(f)
+                    for entry in json_data:
+                        if entry["input_data"] == input_data:
+                            return entry
+                finally:
+                    fcntl.flock(lock_f, fcntl.LOCK_UN)
         return None
 
     def audio_callback(self, audio_path: str, data: dict, **kwargs):

manim_voiceover/services/openai.py

@@ -50,6 +50,7 @@ def __init__(
         voice: str = "alloy",
         model: str = "tts-1-hd",
         transcription_model="base",
+        api_key: str = None,
         **kwargs
     ):
         """
@@ -60,10 +61,13 @@ def __init__(
             model (str, optional): The TTS model to use.
                 See the `API page <https://platform.openai.com/docs/api-reference/audio/createSpeech>`__
                 for all the available options. Defaults to ``"tts-1-hd"``.
+            api_key (str, optional): The OpenAI API key. If not provided, falls back
+                to the ``OPENAI_API_KEY`` environment variable.
         """
         prompt_ask_missing_extras("openai", "openai", "OpenAIService")
         self.voice = voice
         self.model = model
+        self.api_key = api_key
 
         SpeechService.__init__(self, transcription_model=transcription_model, **kwargs)
 
@@ -80,6 +84,13 @@ def generate_from_text(
             raise ValueError("The speed must be between 0.25 and 4.0.")
 
         input_text = remove_bookmarks(text)
+
+        if not input_text or not input_text.strip():
+            raise ValueError(
+                "The input text is empty after removing bookmarks. "
+                "Please provide non-empty text for speech synthesis."
+            )
+
         input_data = {
             "input_text": input_text,
             "service": "openai",
@@ -99,10 +110,12 @@ def generate_from_text(
         else:
             audio_path = path
 
-        if os.getenv("OPENAI_API_KEY") is None:
+        api_key = self.api_key or os.getenv("OPENAI_API_KEY")
+        if api_key is None:
             create_dotenv_openai()
 
-        response = openai.audio.speech.create(
+        client = openai.OpenAI(api_key=api_key)
+        response = client.audio.speech.create(
             model=self.model,
             voice=self.voice,
             input=input_text,

manim_voiceover/services/recorder/__init__.py

@@ -42,7 +42,7 @@ def __init__(
             rate (int, optional): Sampling rate. Defaults to 44100.
             chunk (int, optional): Chunk size. Defaults to 512.
             device_index (int, optional): Device index, if you don't want to choose it every time you render. Defaults to None.
-            transcription_model (str, optional): The `OpenAI Whisper model <https://github.com/openai/whisper#available-models-and-languages>`_ to use for transcription. Defaults to "base".
+            transcription_model (str, optional): The `Whisper model <https://github.com/SYSTRAN/faster-whisper#available-models>`_ to use for transcription. Defaults to "base".
             trim_silence_threshold (float, optional): Threshold for trimming silence in decibels. Defaults to -40.0 dB.
             trim_buffer_start (int, optional): Buffer duration for trimming silence at the start. Defaults to 200 ms.
             trim_buffer_end (int, optional): Buffer duration for trimming silence at the end. Defaults to 200 ms.