perf: cut CPU Whisper latency and prune the catalog by measured WER

app/audio.py

@@ -72,7 +72,10 @@ async def normalize(self, source: Path, destination: Path, maximum_seconds: int)
             destination.unlink(missing_ok=True)
             raise InvalidAudioError(_safe_ffmpeg_message(stderr))
         try:
-            self._validate_wave(destination, maximum_seconds)
+            # Reading the whole PCM file and summing its squares is tens of
+            # milliseconds of pure CPU for a minute of audio. On the event loop
+            # that stalls every other request in flight, so it runs on a worker.
+            await asyncio.to_thread(self._validate_wave, destination, maximum_seconds)
         except Exception:
             destination.unlink(missing_ok=True)
             raise
@@ -105,12 +108,26 @@ def _check_wave_properties(self, recording: wave.Wave_read, maximum_seconds: int
     def _check_silence(self, samples: array[int]) -> None:
         if not samples:
             raise InvalidAudioError("Recording contains no audio frames.")
-        energy = sum(sample * sample for sample in samples)
-        rms = math.sqrt(energy / len(samples))
-        if rms < SILENCE_RMS_THRESHOLD:
+        if _root_mean_square(samples) < SILENCE_RMS_THRESHOLD:
             raise SilentAudioError("Recording appears to be silent.")
 
 
+def _root_mean_square(samples: array[int]) -> float:
+    """RMS amplitude, vectorized when numpy is present.
+
+    numpy arrives with every engine extra but the core install does without it,
+    so the pure-Python sum stays as the fallback rather than becoming a new
+    hard dependency of the gateway.
+    """
+    try:
+        import numpy
+    except ImportError:
+        energy = sum(sample * sample for sample in samples)
+        return math.sqrt(energy / len(samples))
+    block = numpy.frombuffer(samples, dtype=numpy.int16).astype(numpy.float64)
+    return float(numpy.sqrt(numpy.square(block).mean()))
+
+
 def validate_audio_upload_headers(
     content_type: str | None, content_length: int | None, max_bytes: int
 ) -> str:

app/catalog.py

@@ -112,6 +112,24 @@
 MOONSHINE_KO_SIZE_BYTES = 71_815_486
 MOONSHINE_UK_SIZE_BYTES = 141_001_214
 DISTIL_LARGE_V3_SIZE_BYTES = 1_515_408_824
+FASTER_WHISPER_LARGE_V3_TURBO_SIZE_BYTES = 1_621_669_956
+FASTER_WHISPER_LARGE_V3_SIZE_BYTES = 3_090_839_273
+FASTER_WHISPER_MEDIUM_SIZE_BYTES = 1_530_575_217
+FASTER_WHISPER_MEDIUM_EN_SIZE_BYTES = 1_530_460_562
+ACCURATE_QUALITY = "Accurate"
+# Turbo is not the most accurate Whisper — the Open ASR Leaderboard puts it at
+# 6.36 average WER against full Large v3's 5.78 — so it must not claim to be in
+# a picker that also offers Large v3 and the Q5 build of those same weights.
+TURBO_QUALITY = "Large-model accuracy · fast decoder"
+WHISPERKIT_COMPRESSED_LARGE_ID = "whisperkit:openai_whisper-large-v3-v20240930_626MB"
+DISTIL_LARGE_V35_SIZE_BYTES = 1_516_487_390
+WHISPER_TURBO_REPLACEMENT_ID = "whisper.cpp:ggml-large-v3-turbo.bin"
+WHISPER_TURBO_RETIREMENT_REASON = (
+    "Whisper Large v3 Turbo replaces this tier: the same encoder with four decoder layers "
+    "instead of 32, so it is smaller and several times faster. It is not more accurate — the "
+    "Open ASR Leaderboard puts Turbo about 0.6 WER points behind full Large v3 on English "
+    "and up to 2 behind on German — but Medium trails both, so nothing here is given up."
+)
 HINGLISH_MODEL_SIZE_BYTES = 574_041_195
 
 # Qwen3-ASR's upstream card lists 30 languages plus Chinese dialects. The
@@ -253,6 +271,15 @@ def _pin_model(cls, model: CatalogModel, record: Any) -> CatalogModel:
 
 
 class _WhisperModelBuilders:
+    @classmethod
+    def retirement(cls, kwargs: dict[str, Any]) -> dict[str, Any]:
+        """The retirement triple, so each Whisper builder spells it once."""
+        return {
+            "retired": bool(kwargs.get("retired", False)),
+            "replacement_id": kwargs.get("replacement_id"),
+            "retirement_reason": kwargs.get("retirement_reason"),
+        }
+
     @classmethod
     def whisper_language_codes(cls, languages: str) -> tuple[str, ...]:
         return (ENGLISH_LANGUAGE_CODE,) if languages == ENGLISH_ONLY else WHISPER_LANGUAGES
@@ -289,6 +316,7 @@ def whisper_cpp(
             ),
             decoder_language_code=kwargs.get("decoder_language_code"),
             license_name=str(kwargs.get("license_name", "See model source")),  # noqa: WPS226
+            **cls.retirement(kwargs),
         )
 
     @classmethod
@@ -314,6 +342,7 @@ def whisperkit(
             description="Core ML Whisper model optimized for Apple silicon.",
             source="WhisperKit",
             language_codes=cls.whisper_language_codes(cls._languages(args)),
+            **cls.retirement(kwargs),
         )
 
     @classmethod
@@ -333,7 +362,7 @@ def faster_whisper(
             languages=cls._languages(args),
             quality=str(args[2]),
             minimum_ram_gb=float(args[3]),
-            huggingface_repo=cls._faster_whisper_repo(key),
+            huggingface_repo=str(kwargs.get("repository") or cls._faster_whisper_repo(key)),
             huggingface_folder="",
             family="Whisper / CTranslate2",
             description=(
@@ -345,6 +374,7 @@ def faster_whisper(
             language_codes=cls.whisper_language_codes(cls._languages(args)),
             license_name=str(kwargs.get("license_name", "See model source")),
             commercial_use=bool(kwargs.get("commercial_use", True)),
+            **cls.retirement(kwargs),
         )
 
     @classmethod
@@ -381,6 +411,11 @@ def _languages(cls, args: tuple[Any, ...]) -> str:
 
     @classmethod
     def _faster_whisper_repo(cls, key: str) -> str:
+        """Systran publishes the CTranslate2 conversions this engine loads.
+
+        Entries whose conversion lives elsewhere (Whisper Large v3 Turbo has no
+        Systran build) pass `repository=` instead of matching this convention.
+        """
         if key.startswith("distil-"):
             return f"Systran/faster-distil-whisper-{key.removeprefix('distil-')}"
         return f"Systran/faster-whisper-{key}"
@@ -1578,6 +1613,43 @@ def _github_release_page(cls, archive_url: str | None) -> str | None:
     _faster_whisper(
         "small", "faster-whisper Small", _megabytes("484"), MULTILINGUAL, BALANCED_QUALITY, 6
     ),
+    _faster_whisper(
+        "large-v3-turbo",
+        "faster-whisper Large v3 Turbo",
+        FASTER_WHISPER_LARGE_V3_TURBO_SIZE_BYTES,
+        MULTILINGUAL,
+        TURBO_QUALITY,
+        8,
+        repository="deepdml/faster-whisper-large-v3-turbo-ct2",
+        license_name=MIT_LICENSE,
+    ),
+    _faster_whisper(
+        "medium.en",
+        "faster-whisper Medium EN",
+        FASTER_WHISPER_MEDIUM_EN_SIZE_BYTES,
+        ENGLISH_ONLY,
+        ACCURATE_QUALITY,
+        8,
+        license_name=MIT_LICENSE,
+    ),
+    _faster_whisper(
+        "medium",
+        "faster-whisper Medium",
+        FASTER_WHISPER_MEDIUM_SIZE_BYTES,
+        MULTILINGUAL,
+        ACCURATE_QUALITY,
+        8,
+        license_name=MIT_LICENSE,
+    ),
+    _faster_whisper(
+        "large-v3",
+        "faster-whisper Large v3",
+        FASTER_WHISPER_LARGE_V3_SIZE_BYTES,
+        MULTILINGUAL,
+        MOST_ACCURATE_QUALITY,
+        VERY_HIGH_MEMORY_RAM_GB,
+        license_name=MIT_LICENSE,
+    ),
     _faster_whisper(
         "distil-small.en",
         "Distil-Whisper Small EN",
@@ -1594,6 +1666,16 @@ def _github_release_page(cls, archive_url: str | None) -> str | None:
         "Accurate · distilled",
         8,
     ),
+    _faster_whisper(
+        "distil-large-v3.5",
+        "Distil-Whisper Large v3.5",
+        DISTIL_LARGE_V35_SIZE_BYTES,
+        ENGLISH_ONLY,
+        "Most accurate English · distilled",
+        8,
+        repository="distil-whisper/distil-large-v3.5-ct2",
+        license_name=MIT_LICENSE,
+    ),
     _faster_whisper(
         "distil-large-v3",
         "Distil-Whisper Large v3",
@@ -1602,6 +1684,13 @@ def _github_release_page(cls, archive_url: str | None) -> str | None:
         "Most accurate · distilled",
         VERY_HIGH_MEMORY_RAM_GB,
         license_name=MIT_LICENSE,
+        retired=True,
+        replacement_id="faster-whisper:distil-large-v3.5",
+        retirement_reason=(
+            "v3.5 is the same architecture and within 1 MB of the same download, and it is the "
+            "one the Open ASR Leaderboard measures: 5.40 average WER, ahead of full Whisper "
+            "Large v3 at 5.78 and Turbo at 6.36."
+        ),
     ),
     _whisperkit(
         "openai_whisper-tiny", "WhisperKit Tiny", _megabytes("66"), MULTILINGUAL, FASTEST_QUALITY, 4
@@ -1645,14 +1734,25 @@ def _github_release_page(cls, archive_url: str | None) -> str | None:
         MULTILINGUAL,
         BALANCED_QUALITY,
         8,
+        retired=True,
+        replacement_id=WHISPERKIT_COMPRESSED_LARGE_ID,
+        retirement_reason=(
+            "On the same LibriSpeech run this scores 3.95% WER against 2.49% for the compressed "
+            "Large v3, which is 142 MB larger and needs no more memory. The compressed Small "
+            "build stays as the genuinely small option."
+        ),
     ),
     _whisperkit(
         "openai_whisper-large-v3-v20240930_626MB",
         "WhisperKit Large v3 Turbo (compressed)",
         _megabytes("626"),
         MULTILINGUAL,
         MOST_ACCURATE_QUALITY,
-        HIGH_MEMORY_RAM_GB,
+        # Now the only full-quality WhisperKit tier, so it has to be offered to
+        # the 8 GB Macs that used to pick Small. A 626 MB Core ML model is well
+        # within that budget; the old 12 GB floor was inherited from the 1.6 GB
+        # build this entry replaces.
+        8,
     ),
     _whisperkit(
         "openai_whisper-large-v3-v20240930_turbo",
@@ -1661,6 +1761,13 @@ def _github_release_page(cls, archive_url: str | None) -> str | None:
         MULTILINGUAL,
         MOST_ACCURATE_QUALITY,
         VERY_HIGH_MEMORY_RAM_GB,
+        retired=True,
+        replacement_id=WHISPERKIT_COMPRESSED_LARGE_ID,
+        retirement_reason=(
+            "Argmax's own evaluation runs both builds over the 2,620 LibriSpeech utterances: "
+            "2.40% WER here against 2.49% for the compressed build. Eight hundredths of a WER "
+            "point is not worth 984 MB of download and 16 GB of required RAM."
+        ),
     ),
     _whisper_cpp(
         "ggml-tiny.en.bin",
@@ -1700,16 +1807,22 @@ def _github_release_page(cls, archive_url: str | None) -> str | None:
         "whisper.cpp Medium EN",
         _megabytes("1500"),
         ENGLISH_ONLY,
-        "Accurate",
+        ACCURATE_QUALITY,
         HIGH_MEMORY_RAM_GB,
+        retired=True,
+        replacement_id=WHISPER_TURBO_REPLACEMENT_ID,
+        retirement_reason=WHISPER_TURBO_RETIREMENT_REASON,
     ),
     _whisper_cpp(
         "ggml-medium.bin",
         "whisper.cpp Medium",
         _megabytes("1500"),
         MULTILINGUAL,
-        "Accurate",
+        ACCURATE_QUALITY,
         HIGH_MEMORY_RAM_GB,
+        retired=True,
+        replacement_id=WHISPER_TURBO_REPLACEMENT_ID,
+        retirement_reason=WHISPER_TURBO_RETIREMENT_REASON,
     ),
     _whisper_cpp(
         "whisper-medium-q4_1.bin",
@@ -1732,8 +1845,11 @@ def _github_release_page(cls, archive_url: str | None) -> str | None:
         "whisper.cpp Large v3 Turbo",
         _megabytes("1620"),
         MULTILINGUAL,
-        MOST_ACCURATE_QUALITY,
-        VERY_HIGH_MEMORY_RAM_GB,
+        TURBO_QUALITY,
+        # Now the top whisper.cpp tier, so it has to be offered to the machines
+        # the retired Medium entries used to serve. A 1.6 GB f16 model needs
+        # about 2.5 GB resident, which a 12 GB host has to spare.
+        HIGH_MEMORY_RAM_GB,
     ),
     _whisper_cpp(
         "ggml-large-v3-q5_0.bin",
@@ -1796,6 +1912,14 @@ def _github_release_page(cls, archive_url: str | None) -> str | None:
         MULTILINGUAL,
         MOST_ACCURATE_QUALITY,
         24,
+        retired=True,
+        replacement_id="faster-whisper:large-v3",
+        retirement_reason=(
+            "These are the most accurate Whisper weights there are — 5.78 average WER on the "
+            "Open ASR Leaderboard against Turbo's 6.36 — but not at 3 GB and 24 GB of RAM "
+            "through a CLI that reloads them on every request. The faster-whisper entry runs "
+            "the same weights as a resident INT8 model instead."
+        ),
     ),
 )
 
@@ -1833,7 +1957,9 @@ def _high_ram_ids(cls, is_apple: bool) -> set[str]:
         return {
             f"{ENGINE_SHERPA_ONNX}:parakeet-tdt-0.6b-v3-int8",
             f"{ENGINE_SHERPA_ONNX}:parakeet-tdt-0.6b-v2-int8",
-            f"{ENGINE_FASTER_WHISPER}:small",
+            # Turbo is the one Whisper tier a CPU-only host can run at a usable
+            # speed: same encoder as Large v3, four decoder layers instead of 32.
+            f"{ENGINE_FASTER_WHISPER}:large-v3-turbo",
             f"{ENGINE_FASTER_WHISPER}:distil-medium.en",
         }
 

app/engines.py

@@ -183,6 +183,7 @@ def _resolve_fallback(self, rc: runtime_config.RuntimeConfig) -> base.Transcript
             self.settings.whisper_binary,
             cpp_p,
             self.catalog_model_for_path(cpp_p),
+            cpu_threads=rc.cpu_threads,
         )
 
 
@@ -289,6 +290,7 @@ def _build_named(
                 self.settings.whisper_binary,
                 path or self.settings.whisper_model,
                 self.resolver.catalog_model_for_path(path or self.settings.whisper_model),
+                cpu_threads=rc.cpu_threads,
             )
         )