Skip to content

fix: make Cohere Transcribe usable and stop buffered streaming from crawling #54

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Mr-Sunglasses merged 1 commit into from
Sep 6, 2026
Merged

fix: make Cohere Transcribe usable and stop buffered streaming from crawling #54

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 29 additions & 6 deletions app/catalog.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,10 @@
FASTER_WHISPER_MEDIUM_SIZE_BYTES = 1_530_575_217
FASTER_WHISPER_MEDIUM_EN_SIZE_BYTES = 1_530_460_562
ACCURATE_QUALITY = "Accurate"
# Buffered streaming exports pay for their low latency in throughput: they
# re-encode a whole context window per step, so they must not sit in a picker
# under a label like "Fast" beside the batch export of the same weights.
LOW_LATENCY_HIGH_CPU_QUALITY = "Low latency · high CPU"
# Turbo is not the most accurate Whisper — the Open ASR Leaderboard puts it at
# 6.36 average WER against full Large v3's 5.78 — so it must not claim to be in
# a picker that also offers Large v3 and the Q5 build of those same weights.
Expand Down Expand Up @@ -219,6 +223,15 @@ class CatalogModel:
decoder_prompt: str | None = None
apple_silicon_only: bool = False
detects_language_automatically: bool = False
# True when the decoder refuses to run without being told the spoken
# language. Clients must not offer "detect language" for these — the
# request fails outright rather than falling back to a default.
requires_explicit_language: bool = False
# A non-streaming export of the same weights, for models whose streaming
# export is buffered rather than cache-aware and so costs many times more
# compute per second of audio. Used only for whole-file transcription,
# where there is no latency to save; live streaming still uses this model.
batch_twin_id: str | None = None
retired: bool = False
replacement_id: str | None = None
retirement_reason: str | None = None
Expand Down Expand Up @@ -527,6 +540,8 @@ def sherpa_onnx(
detects_language_automatically=bool(
kwargs.get("detects_language_automatically", False)
),
requires_explicit_language=bool(kwargs.get("requires_explicit_language", False)),
batch_twin_id=kwargs.get("batch_twin_id"),
)

@classmethod
Expand Down Expand Up @@ -1356,8 +1371,9 @@ def _github_release_page(cls, archive_url: str | None) -> str | None:
family="Parakeet Unified",
model_type=NEMO_TRANSDUCER_TYPE,
description=(
"Unified FastConformer RNNT for English. INT8 CPU export. The streaming variant uses "
"560 ms model context; end-to-end latency depends on the host. "
"Unified FastConformer RNNT for English. INT8 CPU export. Encodes the whole "
"recording in one pass, which makes it roughly twenty times cheaper than the "
"streaming export of the same weights — prefer it unless you need live partials. "
),
language_codes=ENGLISH_CODES,
license_name="NVIDIA Open Model License",
Expand All @@ -1368,19 +1384,24 @@ def _github_release_page(cls, archive_url: str | None) -> str | None:
"Parakeet Unified English INT8 Streaming 560 ms",
PARAKEET_UNIFIED_STREAMING_SIZE_BYTES,
ENGLISH_ONLY,
"Fast · streaming",
LOW_LATENCY_HIGH_CPU_QUALITY,
4,
huggingface_repo="csukuangfj2/sherpa-onnx-nemo-parakeet-unified-en-0.6b-int8-streaming-560ms",
required_files=(ENCODER_INT8_FILE, DECODER_INT8_FILE, JOINER_INT8_FILE, TOKENS_FILE),
family="Parakeet Unified",
model_type=STREAMING_TRANSDUCER_TYPE,
description=(
"Unified FastConformer RNNT for English. INT8 CPU export. The streaming variant uses "
"560 ms model context; end-to-end latency depends on the host. "
"Unified FastConformer RNNT for English, exported for 560 ms streaming latency. "
"That 560 ms is how long after a word you see it, not how fast it runs: this is a "
"buffered export, so every 160 ms step re-encodes the whole 6.2-second context "
"window. It costs roughly twenty times the compute of the non-streaming export of "
"the same weights and only just keeps up with real time on a busy CPU. Install it "
"for live partials; install the non-streaming variant for recorded audio. "
),
language_codes=ENGLISH_CODES,
license_name="NVIDIA Open Model License",
supports_streaming=True,
batch_twin_id=f"{ENGINE_SHERPA_ONNX}:parakeet-unified-en-0.6b-int8",
),
_sherpa_onnx(
"cohere-transcribe-14-lang-int8",
Expand All @@ -1398,10 +1419,12 @@ def _github_release_page(cls, archive_url: str | None) -> str | None:
),
family="Cohere Transcribe",
model_type=COHERE_TRANSCRIBE_TYPE,
requires_explicit_language=True,
description=(
"Cohere's multilingual speech recognizer through a community INT8 CPU export. Select "
"the spoken language explicitly; automatic language detection and Hindi are not "
"supported. "
"supported. The decoder rejects the request outright rather than guessing, so "
'clients left on "detect language" get an error instead of a transcript. '
),
language_codes=(
ENGLISH_LANGUAGE_CODE,
Expand Down
36 changes: 34 additions & 2 deletions app/engines.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,38 @@ def catalog_selection(
chosen = min(installed, key=lambda model: model.size_bytes)
return chosen.path, self.model_manager.catalog_model(chosen.id)

def sherpa_engine(
self,
selection: tuple[Path | None, catalog.CatalogModel | None],
rc: runtime_config.RuntimeConfig,
) -> sherpa_onnx.SherpaOnnxEngine:
"""Build the sherpa engine, handing it a batch twin when one is installed."""
root, model = selection
twin_root, twin_model = self.batch_twin(model)
return sherpa_onnx.SherpaOnnxEngine(
root,
model,
cpu_threads=rc.cpu_threads,
batch_root=twin_root,
batch_model=twin_model,
)

def batch_twin(
self, model: catalog.CatalogModel | None
) -> tuple[Path | None, catalog.CatalogModel | None]:
"""Where the batch twin lives, whether or not it is installed yet.

The engine re-checks that directory on every use, so handing it the
path the twin *would* occupy is what lets a twin downloaded after the
engine was built start being used -- and one deleted afterwards stop --
without a rebuild that nothing on those paths triggers.
"""
twin_id = model.batch_twin_id if model else None
twin_model = self.model_manager.catalog_model(twin_id) if twin_id else None
if twin_model is None:
return None, None
return self.model_manager.model_path(twin_model), twin_model

def catalog_model_for_path(self, path: Path | None) -> catalog.CatalogModel | None:
if path is None:
return None
Expand Down Expand Up @@ -196,7 +228,7 @@ def _resolve_fallback(self, rc: runtime_config.RuntimeConfig) -> base.Transcript
return mlx_audio.MLXAudioEngine(mlx_sel[0], mlx_sel[1])
shp_sel = self.catalog_selection(rc.sherpa_model, catalog.ENGINE_SHERPA_ONNX)
if shp_sel[0] is not None:
return sherpa_onnx.SherpaOnnxEngine(shp_sel[0], shp_sel[1], cpu_threads=rc.cpu_threads)
return self.sherpa_engine(shp_sel, rc)
fw_p = self.resolve_path(catalog.ENGINE_FASTER_WHISPER, rc)
if fw_p is not None:
return faster_whisper.FasterWhisperEngine(
Expand Down Expand Up @@ -286,7 +318,7 @@ def _build_named(
model_id = rc.sherpa_model if sherpa else rc.mlx_audio_model
sel = self.resolver.catalog_selection(model_id, engine)
return (
sherpa_onnx.SherpaOnnxEngine(sel[0], sel[1], cpu_threads=rc.cpu_threads)
self.resolver.sherpa_engine(sel, rc)
if sherpa
else mlx_audio.MLXAudioEngine(sel[0], sel[1])
)
Expand Down
Loading