fix: make Cohere Transcribe usable and stop buffered streaming from crawling (#54)

Author: Mr-Sunglasses

app/catalog.py

@@ -115,6 +115,10 @@
 FASTER_WHISPER_MEDIUM_SIZE_BYTES = 1_530_575_217
 FASTER_WHISPER_MEDIUM_EN_SIZE_BYTES = 1_530_460_562
 ACCURATE_QUALITY = "Accurate"
+# Buffered streaming exports pay for their low latency in throughput: they
+# re-encode a whole context window per step, so they must not sit in a picker
+# under a label like "Fast" beside the batch export of the same weights.
+LOW_LATENCY_HIGH_CPU_QUALITY = "Low latency · high CPU"
 # Turbo is not the most accurate Whisper — the Open ASR Leaderboard puts it at
 # 6.36 average WER against full Large v3's 5.78 — so it must not claim to be in
 # a picker that also offers Large v3 and the Q5 build of those same weights.
@@ -219,6 +223,15 @@ class CatalogModel:
     decoder_prompt: str | None = None
     apple_silicon_only: bool = False
     detects_language_automatically: bool = False
+    # True when the decoder refuses to run without being told the spoken
+    # language. Clients must not offer "detect language" for these — the
+    # request fails outright rather than falling back to a default.
+    requires_explicit_language: bool = False
+    # A non-streaming export of the same weights, for models whose streaming
+    # export is buffered rather than cache-aware and so costs many times more
+    # compute per second of audio. Used only for whole-file transcription,
+    # where there is no latency to save; live streaming still uses this model.
+    batch_twin_id: str | None = None
     retired: bool = False
     replacement_id: str | None = None
     retirement_reason: str | None = None
@@ -527,6 +540,8 @@ def sherpa_onnx(
             detects_language_automatically=bool(
                 kwargs.get("detects_language_automatically", False)
             ),
+            requires_explicit_language=bool(kwargs.get("requires_explicit_language", False)),
+            batch_twin_id=kwargs.get("batch_twin_id"),
         )
 
     @classmethod
@@ -1356,8 +1371,9 @@ def _github_release_page(cls, archive_url: str | None) -> str | None:
         family="Parakeet Unified",
         model_type=NEMO_TRANSDUCER_TYPE,
         description=(
-            "Unified FastConformer RNNT for English. INT8 CPU export. The streaming variant uses "
-            "560 ms model context; end-to-end latency depends on the host. "
+            "Unified FastConformer RNNT for English. INT8 CPU export. Encodes the whole "
+            "recording in one pass, which makes it roughly twenty times cheaper than the "
+            "streaming export of the same weights — prefer it unless you need live partials. "
         ),
         language_codes=ENGLISH_CODES,
         license_name="NVIDIA Open Model License",
@@ -1368,19 +1384,24 @@ def _github_release_page(cls, archive_url: str | None) -> str | None:
         "Parakeet Unified English INT8 Streaming 560 ms",
         PARAKEET_UNIFIED_STREAMING_SIZE_BYTES,
         ENGLISH_ONLY,
-        "Fast · streaming",
+        LOW_LATENCY_HIGH_CPU_QUALITY,
         4,
         huggingface_repo="csukuangfj2/sherpa-onnx-nemo-parakeet-unified-en-0.6b-int8-streaming-560ms",
         required_files=(ENCODER_INT8_FILE, DECODER_INT8_FILE, JOINER_INT8_FILE, TOKENS_FILE),
         family="Parakeet Unified",
         model_type=STREAMING_TRANSDUCER_TYPE,
         description=(
-            "Unified FastConformer RNNT for English. INT8 CPU export. The streaming variant uses "
-            "560 ms model context; end-to-end latency depends on the host. "
+            "Unified FastConformer RNNT for English, exported for 560 ms streaming latency. "
+            "That 560 ms is how long after a word you see it, not how fast it runs: this is a "
+            "buffered export, so every 160 ms step re-encodes the whole 6.2-second context "
+            "window. It costs roughly twenty times the compute of the non-streaming export of "
+            "the same weights and only just keeps up with real time on a busy CPU. Install it "
+            "for live partials; install the non-streaming variant for recorded audio. "
         ),
         language_codes=ENGLISH_CODES,
         license_name="NVIDIA Open Model License",
         supports_streaming=True,
+        batch_twin_id=f"{ENGINE_SHERPA_ONNX}:parakeet-unified-en-0.6b-int8",
     ),
     _sherpa_onnx(
         "cohere-transcribe-14-lang-int8",
@@ -1398,10 +1419,12 @@ def _github_release_page(cls, archive_url: str | None) -> str | None:
         ),
         family="Cohere Transcribe",
         model_type=COHERE_TRANSCRIBE_TYPE,
+        requires_explicit_language=True,
         description=(
             "Cohere's multilingual speech recognizer through a community INT8 CPU export. Select "
             "the spoken language explicitly; automatic language detection and Hindi are not "
-            "supported. "
+            "supported. The decoder rejects the request outright rather than guessing, so "
+            'clients left on "detect language" get an error instead of a transcript. '
         ),
         language_codes=(
             ENGLISH_LANGUAGE_CODE,

app/engines.py

@@ -120,6 +120,38 @@ def catalog_selection(
         chosen = min(installed, key=lambda model: model.size_bytes)
         return chosen.path, self.model_manager.catalog_model(chosen.id)
 
+    def sherpa_engine(
+        self,
+        selection: tuple[Path | None, catalog.CatalogModel | None],
+        rc: runtime_config.RuntimeConfig,
+    ) -> sherpa_onnx.SherpaOnnxEngine:
+        """Build the sherpa engine, handing it a batch twin when one is installed."""
+        root, model = selection
+        twin_root, twin_model = self.batch_twin(model)
+        return sherpa_onnx.SherpaOnnxEngine(
+            root,
+            model,
+            cpu_threads=rc.cpu_threads,
+            batch_root=twin_root,
+            batch_model=twin_model,
+        )
+
+    def batch_twin(
+        self, model: catalog.CatalogModel | None
+    ) -> tuple[Path | None, catalog.CatalogModel | None]:
+        """Where the batch twin lives, whether or not it is installed yet.
+
+        The engine re-checks that directory on every use, so handing it the
+        path the twin *would* occupy is what lets a twin downloaded after the
+        engine was built start being used -- and one deleted afterwards stop --
+        without a rebuild that nothing on those paths triggers.
+        """
+        twin_id = model.batch_twin_id if model else None
+        twin_model = self.model_manager.catalog_model(twin_id) if twin_id else None
+        if twin_model is None:
+            return None, None
+        return self.model_manager.model_path(twin_model), twin_model
+
     def catalog_model_for_path(self, path: Path | None) -> catalog.CatalogModel | None:
         if path is None:
             return None
@@ -196,7 +228,7 @@ def _resolve_fallback(self, rc: runtime_config.RuntimeConfig) -> base.Transcript
             return mlx_audio.MLXAudioEngine(mlx_sel[0], mlx_sel[1])
         shp_sel = self.catalog_selection(rc.sherpa_model, catalog.ENGINE_SHERPA_ONNX)
         if shp_sel[0] is not None:
-            return sherpa_onnx.SherpaOnnxEngine(shp_sel[0], shp_sel[1], cpu_threads=rc.cpu_threads)
+            return self.sherpa_engine(shp_sel, rc)
         fw_p = self.resolve_path(catalog.ENGINE_FASTER_WHISPER, rc)
         if fw_p is not None:
             return faster_whisper.FasterWhisperEngine(
@@ -286,7 +318,7 @@ def _build_named(
             model_id = rc.sherpa_model if sherpa else rc.mlx_audio_model
             sel = self.resolver.catalog_selection(model_id, engine)
             return (
-                sherpa_onnx.SherpaOnnxEngine(sel[0], sel[1], cpu_threads=rc.cpu_threads)
+                self.resolver.sherpa_engine(sel, rc)
                 if sherpa
                 else mlx_audio.MLXAudioEngine(sel[0], sel[1])
             )