Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 30 additions & 0 deletions app/catalog.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@
VIETNAMESE_LANGUAGE_CODE = "vi"
CHINESE_LANGUAGE_CODE = "zh"
HINDI_LANGUAGE_CODE = "hi"
HINGLISH_ROMAN_LANGUAGE_CODE = "hinglish_roman"
TAGALOG_LANGUAGE_CODE = "tl"
MIT_LICENSE = "MIT"
ENGLISH_ONLY = "English only"
Expand Down Expand Up @@ -111,6 +112,7 @@
MOONSHINE_KO_SIZE_BYTES = 71_815_486
MOONSHINE_UK_SIZE_BYTES = 141_001_214
DISTIL_LARGE_V3_SIZE_BYTES = 1_515_408_824
HINGLISH_MODEL_SIZE_BYTES = 574_041_195

# Qwen3-ASR's upstream card lists 30 languages plus Chinese dialects. The
# dialects are represented by the model's Mandarin/`yue` capability rather
Expand Down Expand Up @@ -181,6 +183,10 @@ class CatalogModel:
file_digests: tuple[tuple[str, str], ...] = ()
model_type: str | None = None
language_codes: tuple[str, ...] = ()
# Some fine-tunes expose an application-level output contract whose wire
# value is not a token understood by the decoder (for example, Roman
# Hinglish is requested as `hinglish_roman` but Whisper expects `hi`).
decoder_language_code: str | None = None
apple_silicon_only: bool = False
detects_language_automatically: bool = False
retired: bool = False
Expand Down Expand Up @@ -281,6 +287,7 @@ def whisper_cpp(
language_codes=tuple(
kwargs.get("language_codes") or cls.whisper_language_codes(cls._languages(args))
),
decoder_language_code=kwargs.get("decoder_language_code"),
license_name=str(kwargs.get("license_name", "See model source")), # noqa: WPS226
)

Expand Down Expand Up @@ -554,6 +561,7 @@ def _validate_sherpa_source(
"haw": "Hawaiian",
"he": "Hebrew",
HINDI_LANGUAGE_CODE: "Hindi",
HINGLISH_ROMAN_LANGUAGE_CODE: "Hinglish — Roman",
CROATIAN_LANGUAGE_CODE: "Croatian",
"ht": "Haitian Creole",
HUNGARIAN_LANGUAGE_CODE: "Hungarian",
Expand Down Expand Up @@ -1759,6 +1767,28 @@ def _github_release_page(cls, archive_url: str | None) -> str | None:
language_codes=(CHINESE_LANGUAGE_CODE, ENGLISH_LANGUAGE_CODE),
license_name=APACHE_LICENSE,
),
_whisper_cpp(
"ggml-apex-hinglish-q5_0.bin",
"Hinglish — Roman (Experimental)",
HINGLISH_MODEL_SIZE_BYTES,
"Hindi + English, Roman script",
"Experimental · Roman output",
4,
download_url=(
"https://huggingface.co/Marquestra/Whisper-Hindi2Hinglish-Apex-GGML/"
"resolve/main/ggml-apex-hinglish-q5_0.bin"
),
family="Whisper / Hinglish",
description=(
"Experimental Whisper Large v3 Turbo fine-tune for mixed Hindi and English. "
"It returns the words as spoken in one Latin script rather than Devanagari "
"or an English translation."
),
source="Whisper-Hindi2Hinglish-Apex",
language_codes=(HINGLISH_ROMAN_LANGUAGE_CODE,),
decoder_language_code=HINDI_LANGUAGE_CODE,
license_name=APACHE_LICENSE,
),
_whisper_cpp(
"ggml-large-v3.bin",
"whisper.cpp Large v3",
Expand Down
18 changes: 16 additions & 2 deletions app/engines.py
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,14 @@ def catalog_selection(
chosen = min(installed, key=lambda model: model.size_bytes)
return chosen.path, self.model_manager.catalog_model(chosen.id)

def catalog_model_for_path(self, path: Path | None) -> catalog.CatalogModel | None:
if path is None:
return None
for installed in self.model_manager.installed():
if installed.path == path:
return self.model_manager.catalog_model(installed.id)
return None

def apply_model(self, rc: runtime_config.RuntimeConfig, model_id: str, path_str: str) -> None:
prefix = model_id.split(":", 1)[0]
if prefix == catalog.ENGINE_MOONSHINE:
Expand Down Expand Up @@ -171,7 +179,11 @@ def _resolve_fallback(self, rc: runtime_config.RuntimeConfig) -> base.Transcript
cpu_threads=rc.cpu_threads,
)
cpp_p = self.resolve_path(catalog.ENGINE_WHISPER_CPP, rc) or self.settings.whisper_model
return whisper_cpp.WhisperCppEngine(self.settings.whisper_binary, cpp_p)
return whisper_cpp.WhisperCppEngine(
self.settings.whisper_binary,
cpp_p,
self.catalog_model_for_path(cpp_p),
)


class _EngineBuilder:
Expand Down Expand Up @@ -274,7 +286,9 @@ def _build_named(
WhisperKitEngine(self.settings.whisperkit_binary, path)
if engine == catalog.ENGINE_WHISPERKIT
else whisper_cpp.WhisperCppEngine(
self.settings.whisper_binary, path or self.settings.whisper_model
self.settings.whisper_binary,
path or self.settings.whisper_model,
self.resolver.catalog_model_for_path(path or self.settings.whisper_model),
)
)

Expand Down
4 changes: 4 additions & 0 deletions app/model_pins.json
Original file line number Diff line number Diff line change
Expand Up @@ -374,6 +374,10 @@
"revision": "5359861c739e955e79d9a303bcbc70fb988958b1",
"sha256": "60ed5bc3dd14eea856493d334349b405782ddcaf0028d4b5df4088345fba2efe"
},
"whisper.cpp:ggml-apex-hinglish-q5_0.bin": {
"revision": "d1de3ff618856e5675c47d3158ca820506fb4d9e",
"sha256": "9d877151b15cec1feb9110cfbc0a3162cf377bcc0ab1935174226f461cf60f13"
},
"whisper.cpp:ggml-base.en.bin": {
"revision": "5359861c739e955e79d9a303bcbc70fb988958b1",
"sha256": "a03779c86df3323075f5e796cb2ce5029f00ec8869eee3fdfb897afe36c6d002"
Expand Down
40 changes: 36 additions & 4 deletions app/models/whisper_cpp.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@
import tempfile
from pathlib import Path

from app.errors import EngineUnavailableError, TranscriptionProcessError
from app import scripts
from app.catalog import CatalogModel
from app.errors import EngineUnavailableError, LanguageUnsupportedError, TranscriptionProcessError
from app.models.base import EngineHealth, TranscriptionOptions
from app.models.warmup import prefetch_model_paths

Expand All @@ -13,9 +15,12 @@


class WhisperCppEngine:
def __init__(self, binary: Path, model: Path) -> None:
def __init__(
self, binary: Path, model: Path, catalog_model: CatalogModel | None = None
) -> None:
self.binary = binary
self.model = model
self.catalog_model = catalog_model

async def health(self) -> EngineHealth:
ready = self.binary.is_file() and self.model.is_file()
Expand All @@ -32,10 +37,37 @@ async def transcribe(self, audio_path: Path, options: TranscriptionOptions) -> s
with tempfile.TemporaryDirectory(prefix="vocagateway-transcript-") as temporary:
output_stem = Path(temporary) / "result"
arguments = _build_arguments(
self.binary, self.model, audio_path, output_stem, options.language
self.binary,
self.model,
audio_path,
output_stem,
self._decoder_language(options.language),
)
await _execute_whisper_cpp(arguments)
return _read_output_text(output_stem.with_suffix(".txt"))
transcript = _read_output_text(output_stem.with_suffix(".txt"))
self._require_fixed_output_script(transcript)
return transcript

def _decoder_language(self, requested: str) -> str:
model = self.catalog_model
if model is None or model.decoder_language_code is None:
return requested
if requested != "auto" and requested not in model.language_codes:
supported = ", ".join(model.language_codes)
raise LanguageUnsupportedError(
f"The selected model supports only {supported}; choose that output mode or Auto."
)
return model.decoder_language_code

def _require_fixed_output_script(self, transcript: str) -> None:
model = self.catalog_model
if model is None or model.decoder_language_code is None or len(model.language_codes) != 1:
return
output_language = model.language_codes[0]
if not scripts.transcript_matches_language(transcript, output_language):
raise LanguageUnsupportedError(
f"The model did not produce the required {output_language} writing system."
)

async def _require_ready(self) -> None:
health = await self.health()
Expand Down
4 changes: 2 additions & 2 deletions app/routes/transcriptions.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@

from app import audio, context, errors, schemas

_LANGUAGE_PATTERN = re.compile(r"^[A-Za-z-]+$|^auto$")
_LANGUAGE_PATTERN = re.compile(r"^(?:[A-Za-z-]+|hinglish_roman)$")
_TRUTHY = frozenset(("true", "1", "yes"))
_MAX_LANGUAGE_LENGTH = 20
_READ_CHUNK_BYTES = 65_536
Expand Down Expand Up @@ -129,7 +129,7 @@ def language(cls, language: str | None) -> str:
raise errors.APIProblem(
status.HTTP_422_UNPROCESSABLE_CONTENT,
"invalid_language",
"Language must be auto or a language tag.",
"Language must be auto, a language tag, or hinglish_roman.",
)
return language_value

Expand Down
4 changes: 3 additions & 1 deletion app/schemas.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,9 @@ class CreateSessionRequest(BaseModel):

client_session_id: UUID
language: str = Field(
default=AUTO_ENGINE, max_length=MAXIMUM_LANGUAGE_TAG_LENGTH, pattern=r"^[A-Za-z-]+$|^auto$"
default=AUTO_ENGINE,
max_length=MAXIMUM_LANGUAGE_TAG_LENGTH,
pattern=r"^(?:[A-Za-z-]+|hinglish_roman)$",
)
style: Literal[
"raw",
Expand Down
10 changes: 9 additions & 1 deletion app/scripts.py
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,7 @@
# instead of being wrongly held to Latin.
_LATIN_LANGUAGES = frozenset(
(
"hinglish_roman",
"en",
"es",
"fr",
Expand Down Expand Up @@ -175,6 +176,10 @@
# code-switching keeps far more of the base script than that — Hinglish rarely
# drops below a third — so this sits well below any real transcript.
_MINIMUM_EXPECTED_SHARE = 0.15
# A fixed-script output mode is a stronger promise than an ordinary spoken
# language. Code-switching is valid Hindi, but any non-Latin letter means the
# Roman Hinglish contract was not met.
_STRICT_SCRIPT_LANGUAGES = frozenset(("hinglish_roman",))


def _script_of(character: str) -> str:
Expand All @@ -201,6 +206,8 @@ def transcript_matches_language(text: str, language: str) -> bool:
with no letters at all. Otherwise the expected script has to account for at
least `_MINIMUM_EXPECTED_SHARE` of the letters, which admits code-switching
while rejecting text transliterated into another writing system entirely.
Fixed-script output contracts require every letter to use their declared
script.
"""
if language == "auto":
return True
Expand All @@ -211,4 +218,5 @@ def transcript_matches_language(text: str, language: str) -> bool:
if not letters:
return True
matching = sum(1 for character in letters if _script_of(character) in expected)
return matching / len(letters) >= _MINIMUM_EXPECTED_SHARE
minimum_share = 1.0 if language.lower() in _STRICT_SCRIPT_LANGUAGES else _MINIMUM_EXPECTED_SHARE
return matching / len(letters) >= minimum_share
14 changes: 12 additions & 2 deletions docs/models.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

# Models and languages

Every model in the catalog (65 of them), what it speaks, and which models cover a given language. Generated from `app/catalog.py`, so it always matches the catalog the gateway actually ships.
Every model in the catalog (66 of them), what it speaks, and which models cover a given language. Generated from `app/catalog.py`, so it always matches the catalog the gateway actually ships.

The WebUI Models tab shows the same information per card, with a language filter. Use this page to pick a model before installing anything.

Expand Down Expand Up @@ -98,6 +98,7 @@ GGML models through the standalone `whisper-cli` binary.
| whisper.cpp Large v3 Turbo | 1.62 GB | [100 languages](#language-set-100-0b8e4ee5) | — | See model source |
| Whisper Large v3 Q5 | 1.08 GB | [100 languages](#language-set-100-0b8e4ee5) | — | See model source |
| Breeze ASR Q5 | 1.08 GB | Mandarin Chinese, English | — | Apache 2.0 |
| Hinglish — Roman (Experimental) | 574 MB | Hinglish — Roman | — | Apache 2.0 |
| whisper.cpp Large v3 | 3.00 GB | [100 languages](#language-set-100-0b8e4ee5) | — | See model source |

### whisperkit
Expand Down Expand Up @@ -174,7 +175,7 @@ Afrikaans, Amharic, Arabic, Assamese, Azerbaijani, Bashkir, Belarusian, Bulgaria

## Language index

108 languages, alphabetically. Expand one to see every model that covers it. A model marked `auto language` will not let you pin this language explicitly — it decides for itself.
109 languages, alphabetically. Expand one to see every model that covers it. A model marked `auto language` will not let you pin this language explicitly — it decides for itself.

<details>
<summary><strong>Afrikaans</strong> (<code>af</code>) — 18 models</summary>
Expand Down Expand Up @@ -1235,6 +1236,15 @@ Afrikaans, Amharic, Arabic, Assamese, Azerbaijani, Bashkir, Belarusian, Bulgaria

</details>

<details>
<summary><strong>Hinglish — Roman</strong> (<code>hinglish_roman</code>) — 1 models</summary>

| Model | Engine | Download | Flags |
| --- | --- | ---: | --- |
| Hinglish — Roman (Experimental) | whisper.cpp | 574 MB | — |

</details>

<details>
<summary><strong>Hungarian</strong> (<code>hu</code>) — 24 models</summary>

Expand Down
9 changes: 9 additions & 0 deletions tests/test_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
from app.errors import LanguageUnsupportedError
from app.main import create_app
from app.models.base import EngineHealth, TranscriptionOptions
from app.schemas import CreateSessionRequest

TEST_AUDIO_SIZE = 200
TEST_AUDIO_BYTES = b"x" * TEST_AUDIO_SIZE
Expand Down Expand Up @@ -53,6 +54,14 @@ async def transcribe(self, audio_path: Path, options: TranscriptionOptions) -> s
)


def test_session_schema_accepts_roman_hinglish_output_contract() -> None:
request = CreateSessionRequest(client_session_id=uuid4(), language="hinglish_roman")
assert request.language == "hinglish_roman"

with pytest.raises(ValueError):
CreateSessionRequest(client_session_id=uuid4(), language="not_a_language")


async def test_unsupported_language_is_reported_a_d68a4(
settings: Settings, authorization: dict[str, str], audio_bytes: bytes
) -> None:
Expand Down
25 changes: 25 additions & 0 deletions tests/test_engines.py
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,31 @@ def test_model_selection_builds_new_engine_aa(
assert getattr(RuntimeConfig.load(config_path), config_field) == catalog_model.id


def test_whisper_model_selection_keeps_catalog_output_contract(tmp_path: Path) -> None:
catalog_model = CatalogModel(
id="whisper.cpp:hinglish",
engine="whisper.cpp",
key="hinglish.bin",
label="Hinglish",
size_bytes=1,
languages="Hindi + English, Roman script",
quality="Experimental",
minimum_ram_gb=4,
language_codes=("hinglish_roman",),
decoder_language_code="hi",
)
manager = ModelManager(tmp_path / MODELS_DIRECTORY, catalog=(catalog_model,))
model_path = manager.model_path(catalog_model)
model_path.parent.mkdir(parents=True)
model_path.write_bytes(b"model")
settings = _settings(tmp_path)
engines = EngineManager(settings, RuntimeConfig(), tmp_path / "config.json", manager)

engines.select_model(catalog_model.id)

assert getattr(engines.current(), "catalog_model", None) == catalog_model


@pytest.mark.parametrize(
("engine", "linux", "intel_mac", "apple_silicon"),
[
Expand Down
15 changes: 15 additions & 0 deletions tests/test_model_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -354,6 +354,21 @@ def test_catalog_includes_the_newer_apple_s_ed423() -> None:
assert model.huggingface_folder == ""


def test_catalog_includes_the_roman_hinglish_model() -> None:
entries = {model.id: model for model in DEFAULT_CATALOG}
model = entries["whisper.cpp:ggml-apex-hinglish-q5_0.bin"]

assert model.download_url == (
"https://huggingface.co/Marquestra/Whisper-Hindi2Hinglish-Apex-GGML/"
"resolve/d1de3ff618856e5675c47d3158ca820506fb4d9e/ggml-apex-hinglish-q5_0.bin"
)
assert model.size_bytes == 574_041_195
assert model.language_codes == ("hinglish_roman",)
assert model.decoder_language_code == "hi"
assert model.license_name == "Apache 2.0"
assert model.sha256 == ("9d877151b15cec1feb9110cfbc0a3162cf377bcc0ab1935174226f461cf60f13")


def test_every_catalog_model_has_a_download_f979c() -> None:
for model in DEFAULT_CATALOG:
if model.engine == "moonshine":
Expand Down
6 changes: 6 additions & 0 deletions tests/test_scripts.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,12 @@ def test_uncertain_cases_pass_rather_than_fail() -> None:
assert transcript_matches_language("私は元気です", "zh") is True


def test_roman_hinglish_uses_the_latin_script_contract() -> None:
assert transcript_matches_language("Aaj mujhe office jaana hai", "hinglish_roman") is True
assert transcript_matches_language("आज मुझे जाना है", "hinglish_roman") is False
assert transcript_matches_language("Aaj office में meeting hai", "hinglish_roman") is False


@pytest.mark.parametrize(
("text", "language"),
[
Expand Down
Loading