diff --git a/app/catalog.py b/app/catalog.py index 4bf719e..f6fe4bf 100644 --- a/app/catalog.py +++ b/app/catalog.py @@ -60,6 +60,7 @@ VIETNAMESE_LANGUAGE_CODE = "vi" CHINESE_LANGUAGE_CODE = "zh" HINDI_LANGUAGE_CODE = "hi" +HINGLISH_ROMAN_LANGUAGE_CODE = "hinglish_roman" TAGALOG_LANGUAGE_CODE = "tl" MIT_LICENSE = "MIT" ENGLISH_ONLY = "English only" @@ -111,6 +112,7 @@ MOONSHINE_KO_SIZE_BYTES = 71_815_486 MOONSHINE_UK_SIZE_BYTES = 141_001_214 DISTIL_LARGE_V3_SIZE_BYTES = 1_515_408_824 +HINGLISH_MODEL_SIZE_BYTES = 574_041_195 # Qwen3-ASR's upstream card lists 30 languages plus Chinese dialects. The # dialects are represented by the model's Mandarin/`yue` capability rather @@ -181,6 +183,10 @@ class CatalogModel: file_digests: tuple[tuple[str, str], ...] = () model_type: str | None = None language_codes: tuple[str, ...] = () + # Some fine-tunes expose an application-level output contract whose wire + # value is not a token understood by the decoder (for example, Roman + # Hinglish is requested as `hinglish_roman` but Whisper expects `hi`). + decoder_language_code: str | None = None apple_silicon_only: bool = False detects_language_automatically: bool = False retired: bool = False @@ -281,6 +287,7 @@ def whisper_cpp( language_codes=tuple( kwargs.get("language_codes") or cls.whisper_language_codes(cls._languages(args)) ), + decoder_language_code=kwargs.get("decoder_language_code"), license_name=str(kwargs.get("license_name", "See model source")), # noqa: WPS226 ) @@ -554,6 +561,7 @@ def _validate_sherpa_source( "haw": "Hawaiian", "he": "Hebrew", HINDI_LANGUAGE_CODE: "Hindi", + HINGLISH_ROMAN_LANGUAGE_CODE: "Hinglish — Roman", CROATIAN_LANGUAGE_CODE: "Croatian", "ht": "Haitian Creole", HUNGARIAN_LANGUAGE_CODE: "Hungarian", @@ -1759,6 +1767,28 @@ def _github_release_page(cls, archive_url: str | None) -> str | None: language_codes=(CHINESE_LANGUAGE_CODE, ENGLISH_LANGUAGE_CODE), license_name=APACHE_LICENSE, ), + _whisper_cpp( + "ggml-apex-hinglish-q5_0.bin", + "Hinglish — Roman (Experimental)", + HINGLISH_MODEL_SIZE_BYTES, + "Hindi + English, Roman script", + "Experimental · Roman output", + 4, + download_url=( + "https://huggingface.co/Marquestra/Whisper-Hindi2Hinglish-Apex-GGML/" + "resolve/main/ggml-apex-hinglish-q5_0.bin" + ), + family="Whisper / Hinglish", + description=( + "Experimental Whisper Large v3 Turbo fine-tune for mixed Hindi and English. " + "It returns the words as spoken in one Latin script rather than Devanagari " + "or an English translation." + ), + source="Whisper-Hindi2Hinglish-Apex", + language_codes=(HINGLISH_ROMAN_LANGUAGE_CODE,), + decoder_language_code=HINDI_LANGUAGE_CODE, + license_name=APACHE_LICENSE, + ), _whisper_cpp( "ggml-large-v3.bin", "whisper.cpp Large v3", diff --git a/app/engines.py b/app/engines.py index 1e2e62b..c1b41d9 100644 --- a/app/engines.py +++ b/app/engines.py @@ -94,6 +94,14 @@ def catalog_selection( chosen = min(installed, key=lambda model: model.size_bytes) return chosen.path, self.model_manager.catalog_model(chosen.id) + def catalog_model_for_path(self, path: Path | None) -> catalog.CatalogModel | None: + if path is None: + return None + for installed in self.model_manager.installed(): + if installed.path == path: + return self.model_manager.catalog_model(installed.id) + return None + def apply_model(self, rc: runtime_config.RuntimeConfig, model_id: str, path_str: str) -> None: prefix = model_id.split(":", 1)[0] if prefix == catalog.ENGINE_MOONSHINE: @@ -171,7 +179,11 @@ def _resolve_fallback(self, rc: runtime_config.RuntimeConfig) -> base.Transcript cpu_threads=rc.cpu_threads, ) cpp_p = self.resolve_path(catalog.ENGINE_WHISPER_CPP, rc) or self.settings.whisper_model - return whisper_cpp.WhisperCppEngine(self.settings.whisper_binary, cpp_p) + return whisper_cpp.WhisperCppEngine( + self.settings.whisper_binary, + cpp_p, + self.catalog_model_for_path(cpp_p), + ) class _EngineBuilder: @@ -274,7 +286,9 @@ def _build_named( WhisperKitEngine(self.settings.whisperkit_binary, path) if engine == catalog.ENGINE_WHISPERKIT else whisper_cpp.WhisperCppEngine( - self.settings.whisper_binary, path or self.settings.whisper_model + self.settings.whisper_binary, + path or self.settings.whisper_model, + self.resolver.catalog_model_for_path(path or self.settings.whisper_model), ) ) diff --git a/app/model_pins.json b/app/model_pins.json index 0a27271..3a52928 100644 --- a/app/model_pins.json +++ b/app/model_pins.json @@ -374,6 +374,10 @@ "revision": "5359861c739e955e79d9a303bcbc70fb988958b1", "sha256": "60ed5bc3dd14eea856493d334349b405782ddcaf0028d4b5df4088345fba2efe" }, + "whisper.cpp:ggml-apex-hinglish-q5_0.bin": { + "revision": "d1de3ff618856e5675c47d3158ca820506fb4d9e", + "sha256": "9d877151b15cec1feb9110cfbc0a3162cf377bcc0ab1935174226f461cf60f13" + }, "whisper.cpp:ggml-base.en.bin": { "revision": "5359861c739e955e79d9a303bcbc70fb988958b1", "sha256": "a03779c86df3323075f5e796cb2ce5029f00ec8869eee3fdfb897afe36c6d002" diff --git a/app/models/whisper_cpp.py b/app/models/whisper_cpp.py index ee99de0..3e709d9 100644 --- a/app/models/whisper_cpp.py +++ b/app/models/whisper_cpp.py @@ -4,7 +4,9 @@ import tempfile from pathlib import Path -from app.errors import EngineUnavailableError, TranscriptionProcessError +from app import scripts +from app.catalog import CatalogModel +from app.errors import EngineUnavailableError, LanguageUnsupportedError, TranscriptionProcessError from app.models.base import EngineHealth, TranscriptionOptions from app.models.warmup import prefetch_model_paths @@ -13,9 +15,12 @@ class WhisperCppEngine: - def __init__(self, binary: Path, model: Path) -> None: + def __init__( + self, binary: Path, model: Path, catalog_model: CatalogModel | None = None + ) -> None: self.binary = binary self.model = model + self.catalog_model = catalog_model async def health(self) -> EngineHealth: ready = self.binary.is_file() and self.model.is_file() @@ -32,10 +37,37 @@ async def transcribe(self, audio_path: Path, options: TranscriptionOptions) -> s with tempfile.TemporaryDirectory(prefix="vocagateway-transcript-") as temporary: output_stem = Path(temporary) / "result" arguments = _build_arguments( - self.binary, self.model, audio_path, output_stem, options.language + self.binary, + self.model, + audio_path, + output_stem, + self._decoder_language(options.language), ) await _execute_whisper_cpp(arguments) - return _read_output_text(output_stem.with_suffix(".txt")) + transcript = _read_output_text(output_stem.with_suffix(".txt")) + self._require_fixed_output_script(transcript) + return transcript + + def _decoder_language(self, requested: str) -> str: + model = self.catalog_model + if model is None or model.decoder_language_code is None: + return requested + if requested != "auto" and requested not in model.language_codes: + supported = ", ".join(model.language_codes) + raise LanguageUnsupportedError( + f"The selected model supports only {supported}; choose that output mode or Auto." + ) + return model.decoder_language_code + + def _require_fixed_output_script(self, transcript: str) -> None: + model = self.catalog_model + if model is None or model.decoder_language_code is None or len(model.language_codes) != 1: + return + output_language = model.language_codes[0] + if not scripts.transcript_matches_language(transcript, output_language): + raise LanguageUnsupportedError( + f"The model did not produce the required {output_language} writing system." + ) async def _require_ready(self) -> None: health = await self.health() diff --git a/app/routes/transcriptions.py b/app/routes/transcriptions.py index e54a322..1f838a6 100644 --- a/app/routes/transcriptions.py +++ b/app/routes/transcriptions.py @@ -13,7 +13,7 @@ from app import audio, context, errors, schemas -_LANGUAGE_PATTERN = re.compile(r"^[A-Za-z-]+$|^auto$") +_LANGUAGE_PATTERN = re.compile(r"^(?:[A-Za-z-]+|hinglish_roman)$") _TRUTHY = frozenset(("true", "1", "yes")) _MAX_LANGUAGE_LENGTH = 20 _READ_CHUNK_BYTES = 65_536 @@ -129,7 +129,7 @@ def language(cls, language: str | None) -> str: raise errors.APIProblem( status.HTTP_422_UNPROCESSABLE_CONTENT, "invalid_language", - "Language must be auto or a language tag.", + "Language must be auto, a language tag, or hinglish_roman.", ) return language_value diff --git a/app/schemas.py b/app/schemas.py index f87dfa6..609ed1e 100644 --- a/app/schemas.py +++ b/app/schemas.py @@ -20,7 +20,9 @@ class CreateSessionRequest(BaseModel): client_session_id: UUID language: str = Field( - default=AUTO_ENGINE, max_length=MAXIMUM_LANGUAGE_TAG_LENGTH, pattern=r"^[A-Za-z-]+$|^auto$" + default=AUTO_ENGINE, + max_length=MAXIMUM_LANGUAGE_TAG_LENGTH, + pattern=r"^(?:[A-Za-z-]+|hinglish_roman)$", ) style: Literal[ "raw", diff --git a/app/scripts.py b/app/scripts.py index 8a950e4..2fd0264 100644 --- a/app/scripts.py +++ b/app/scripts.py @@ -108,6 +108,7 @@ # instead of being wrongly held to Latin. _LATIN_LANGUAGES = frozenset( ( + "hinglish_roman", "en", "es", "fr", @@ -175,6 +176,10 @@ # code-switching keeps far more of the base script than that — Hinglish rarely # drops below a third — so this sits well below any real transcript. _MINIMUM_EXPECTED_SHARE = 0.15 +# A fixed-script output mode is a stronger promise than an ordinary spoken +# language. Code-switching is valid Hindi, but any non-Latin letter means the +# Roman Hinglish contract was not met. +_STRICT_SCRIPT_LANGUAGES = frozenset(("hinglish_roman",)) def _script_of(character: str) -> str: @@ -201,6 +206,8 @@ def transcript_matches_language(text: str, language: str) -> bool: with no letters at all. Otherwise the expected script has to account for at least `_MINIMUM_EXPECTED_SHARE` of the letters, which admits code-switching while rejecting text transliterated into another writing system entirely. + Fixed-script output contracts require every letter to use their declared + script. """ if language == "auto": return True @@ -211,4 +218,5 @@ def transcript_matches_language(text: str, language: str) -> bool: if not letters: return True matching = sum(1 for character in letters if _script_of(character) in expected) - return matching / len(letters) >= _MINIMUM_EXPECTED_SHARE + minimum_share = 1.0 if language.lower() in _STRICT_SCRIPT_LANGUAGES else _MINIMUM_EXPECTED_SHARE + return matching / len(letters) >= minimum_share diff --git a/docs/models.md b/docs/models.md index 0b3ddf8..4552deb 100644 --- a/docs/models.md +++ b/docs/models.md @@ -2,7 +2,7 @@ # Models and languages -Every model in the catalog (65 of them), what it speaks, and which models cover a given language. Generated from `app/catalog.py`, so it always matches the catalog the gateway actually ships. +Every model in the catalog (66 of them), what it speaks, and which models cover a given language. Generated from `app/catalog.py`, so it always matches the catalog the gateway actually ships. The WebUI Models tab shows the same information per card, with a language filter. Use this page to pick a model before installing anything. @@ -98,6 +98,7 @@ GGML models through the standalone `whisper-cli` binary. | whisper.cpp Large v3 Turbo | 1.62 GB | [100 languages](#language-set-100-0b8e4ee5) | — | See model source | | Whisper Large v3 Q5 | 1.08 GB | [100 languages](#language-set-100-0b8e4ee5) | — | See model source | | Breeze ASR Q5 | 1.08 GB | Mandarin Chinese, English | — | Apache 2.0 | +| Hinglish — Roman (Experimental) | 574 MB | Hinglish — Roman | — | Apache 2.0 | | whisper.cpp Large v3 | 3.00 GB | [100 languages](#language-set-100-0b8e4ee5) | — | See model source | ### whisperkit @@ -174,7 +175,7 @@ Afrikaans, Amharic, Arabic, Assamese, Azerbaijani, Bashkir, Belarusian, Bulgaria ## Language index -108 languages, alphabetically. Expand one to see every model that covers it. A model marked `auto language` will not let you pin this language explicitly — it decides for itself. +109 languages, alphabetically. Expand one to see every model that covers it. A model marked `auto language` will not let you pin this language explicitly — it decides for itself.
Afrikaans (af) — 18 models @@ -1235,6 +1236,15 @@ Afrikaans, Amharic, Arabic, Assamese, Azerbaijani, Bashkir, Belarusian, Bulgaria
+
+Hinglish — Roman (hinglish_roman) — 1 models + +| Model | Engine | Download | Flags | +| --- | --- | ---: | --- | +| Hinglish — Roman (Experimental) | whisper.cpp | 574 MB | — | + +
+
Hungarian (hu) — 24 models diff --git a/tests/test_api.py b/tests/test_api.py index 9e71678..f060816 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -20,6 +20,7 @@ from app.errors import LanguageUnsupportedError from app.main import create_app from app.models.base import EngineHealth, TranscriptionOptions +from app.schemas import CreateSessionRequest TEST_AUDIO_SIZE = 200 TEST_AUDIO_BYTES = b"x" * TEST_AUDIO_SIZE @@ -53,6 +54,14 @@ async def transcribe(self, audio_path: Path, options: TranscriptionOptions) -> s ) +def test_session_schema_accepts_roman_hinglish_output_contract() -> None: + request = CreateSessionRequest(client_session_id=uuid4(), language="hinglish_roman") + assert request.language == "hinglish_roman" + + with pytest.raises(ValueError): + CreateSessionRequest(client_session_id=uuid4(), language="not_a_language") + + async def test_unsupported_language_is_reported_a_d68a4( settings: Settings, authorization: dict[str, str], audio_bytes: bytes ) -> None: diff --git a/tests/test_engines.py b/tests/test_engines.py index 5ff3620..2410401 100644 --- a/tests/test_engines.py +++ b/tests/test_engines.py @@ -114,6 +114,31 @@ def test_model_selection_builds_new_engine_aa( assert getattr(RuntimeConfig.load(config_path), config_field) == catalog_model.id +def test_whisper_model_selection_keeps_catalog_output_contract(tmp_path: Path) -> None: + catalog_model = CatalogModel( + id="whisper.cpp:hinglish", + engine="whisper.cpp", + key="hinglish.bin", + label="Hinglish", + size_bytes=1, + languages="Hindi + English, Roman script", + quality="Experimental", + minimum_ram_gb=4, + language_codes=("hinglish_roman",), + decoder_language_code="hi", + ) + manager = ModelManager(tmp_path / MODELS_DIRECTORY, catalog=(catalog_model,)) + model_path = manager.model_path(catalog_model) + model_path.parent.mkdir(parents=True) + model_path.write_bytes(b"model") + settings = _settings(tmp_path) + engines = EngineManager(settings, RuntimeConfig(), tmp_path / "config.json", manager) + + engines.select_model(catalog_model.id) + + assert getattr(engines.current(), "catalog_model", None) == catalog_model + + @pytest.mark.parametrize( ("engine", "linux", "intel_mac", "apple_silicon"), [ diff --git a/tests/test_model_manager.py b/tests/test_model_manager.py index f8522b2..d00d601 100644 --- a/tests/test_model_manager.py +++ b/tests/test_model_manager.py @@ -354,6 +354,21 @@ def test_catalog_includes_the_newer_apple_s_ed423() -> None: assert model.huggingface_folder == "" +def test_catalog_includes_the_roman_hinglish_model() -> None: + entries = {model.id: model for model in DEFAULT_CATALOG} + model = entries["whisper.cpp:ggml-apex-hinglish-q5_0.bin"] + + assert model.download_url == ( + "https://huggingface.co/Marquestra/Whisper-Hindi2Hinglish-Apex-GGML/" + "resolve/d1de3ff618856e5675c47d3158ca820506fb4d9e/ggml-apex-hinglish-q5_0.bin" + ) + assert model.size_bytes == 574_041_195 + assert model.language_codes == ("hinglish_roman",) + assert model.decoder_language_code == "hi" + assert model.license_name == "Apache 2.0" + assert model.sha256 == ("9d877151b15cec1feb9110cfbc0a3162cf377bcc0ab1935174226f461cf60f13") + + def test_every_catalog_model_has_a_download_f979c() -> None: for model in DEFAULT_CATALOG: if model.engine == "moonshine": diff --git a/tests/test_scripts.py b/tests/test_scripts.py index 96d4998..c4d1644 100644 --- a/tests/test_scripts.py +++ b/tests/test_scripts.py @@ -55,6 +55,12 @@ def test_uncertain_cases_pass_rather_than_fail() -> None: assert transcript_matches_language("私は元気です", "zh") is True +def test_roman_hinglish_uses_the_latin_script_contract() -> None: + assert transcript_matches_language("Aaj mujhe office jaana hai", "hinglish_roman") is True + assert transcript_matches_language("आज मुझे जाना है", "hinglish_roman") is False + assert transcript_matches_language("Aaj office में meeting hai", "hinglish_roman") is False + + @pytest.mark.parametrize( ("text", "language"), [ diff --git a/tests/test_transcriptions.py b/tests/test_transcriptions.py index 8b190d4..4892d21 100644 --- a/tests/test_transcriptions.py +++ b/tests/test_transcriptions.py @@ -20,6 +20,7 @@ from app.errors import EngineUnavailableError from app.main import create_app from app.models.base import EngineHealth, TranscriptionOptions +from app.routes.transcriptions import _AudioForm TRANSCRIPTIONS = "/v1/audio/transcriptions" MULTIPART_CONTENT_TYPE = "multipart/form-data; boundary=----x" @@ -50,6 +51,11 @@ async def transcribe(self, audio_path: Path, options: TranscriptionOptions) -> s raise EngineUnavailableError("The local engine is not available.") +def test_audio_form_accepts_roman_hinglish_output_contract() -> None: + assert _AudioForm.language("hinglish_roman") == "hinglish_roman" + assert _AudioForm._invalid_language("not_a_language") is True + + async def test_transcriptions_require_a_bearer_token( client: httpx.AsyncClient, audio_bytes: bytes ) -> None: diff --git a/tests/test_whisper_cpp.py b/tests/test_whisper_cpp.py index cb650e7..2f4eb2b 100644 --- a/tests/test_whisper_cpp.py +++ b/tests/test_whisper_cpp.py @@ -4,7 +4,8 @@ import pytest -from app.errors import EngineUnavailableError, TranscriptionProcessError +from app.catalog import CatalogModel +from app.errors import EngineUnavailableError, LanguageUnsupportedError, TranscriptionProcessError from app.models.base import TranscriptionOptions from app.models.whisper_cpp import WhisperCppEngine @@ -88,6 +89,89 @@ async def test_transcribe_writes_the_output_stem_aaa( assert "-l" not in recorded +async def test_transcribe_uses_catalog_decoder_language_for_output_contract( + tmp_path: Path, +) -> None: + binary = tmp_path / WHISPER_BINARY_NAME + _write_binary( + binary, + r"""#!/bin/sh +printf '%s\n' "$@" > "$0.args" +of="" +while [ "$#" -gt 0 ]; do + case "$1" in + -of) of="$2"; shift 2 ;; + *) shift ;; + esac +done +printf '%s' "aaj office hai" > "$of.txt" +""", + ) + model = tmp_path / MODEL_FILE_NAME + model.write_bytes(MODEL_BYTES) + audio = tmp_path / AUDIO_FILE_NAME + audio.write_bytes(AUDIO_BYTES) + catalog_model = CatalogModel( + id="whisper.cpp:hinglish", + engine="whisper.cpp", + key=MODEL_FILE_NAME, + label="Hinglish", + size_bytes=1, + languages="Hindi + English, Roman script", + quality="Experimental", + minimum_ram_gb=4, + language_codes=("hinglish_roman",), + decoder_language_code="hi", + ) + + engine = WhisperCppEngine(binary, model, catalog_model) + await engine.transcribe(audio, TranscriptionOptions("hinglish_roman", RAW_STYLE)) + + recorded = (tmp_path / "whisper-cli.args").read_text(encoding="utf-8").splitlines() + assert recorded[recorded.index("-l") + 1] == "hi" + + with pytest.raises(LanguageUnsupportedError, match="only hinglish_roman"): + await engine.transcribe(audio, TranscriptionOptions("en", RAW_STYLE)) + + +async def test_fixed_output_contract_rejects_devanagari_leakage(tmp_path: Path) -> None: + binary = tmp_path / WHISPER_BINARY_NAME + _write_binary( + binary, + r"""#!/bin/sh +of="" +while [ "$#" -gt 0 ]; do + case "$1" in + -of) of="$2"; shift 2 ;; + *) shift ;; + esac +done +printf '%s' "Aaj office में meeting hai" > "$of.txt" +""", + ) + model = tmp_path / MODEL_FILE_NAME + model.write_bytes(MODEL_BYTES) + audio = tmp_path / AUDIO_FILE_NAME + audio.write_bytes(AUDIO_BYTES) + catalog_model = CatalogModel( + id="whisper.cpp:hinglish", + engine="whisper.cpp", + key=MODEL_FILE_NAME, + label="Hinglish", + size_bytes=1, + languages="Hindi + English, Roman script", + quality="Experimental", + minimum_ram_gb=4, + language_codes=("hinglish_roman",), + decoder_language_code="hi", + ) + + with pytest.raises(LanguageUnsupportedError, match="required hinglish_roman"): + await WhisperCppEngine(binary, model, catalog_model).transcribe( + audio, TranscriptionOptions("auto", RAW_STYLE) + ) + + async def test_transcribe_raises_when_the_engine_aaaa(tmp_path: Path) -> None: engine = WhisperCppEngine(tmp_path / "missing-cli", tmp_path / "missing-model.bin")