sensein
diff --git a/‎pyproject.toml‎
Lines changed: 4 additions & 2 deletions b/‎pyproject.toml‎
Lines changed: 4 additions & 2 deletions
diff --git a/‎scripts/analyze_audio.py‎
Lines changed: 1544 additions & 0 deletions b/‎scripts/analyze_audio.py‎
Lines changed: 1544 additions & 0 deletions
diff --git a/‎scripts/generate_model_registry.py‎
Lines changed: 1 addition & 1 deletion b/‎scripts/generate_model_registry.py‎
Lines changed: 1 addition & 1 deletion
diff --git a/‎scripts/profile_model_tiers.py‎
Lines changed: 5 additions & 20 deletions b/‎scripts/profile_model_tiers.py‎
Lines changed: 5 additions & 20 deletions
diff --git a/‎specs/20260506-154425-audio-analysis-asr-extensions/tasks.md‎
Lines changed: 251 additions & 0 deletions b/‎specs/20260506-154425-audio-analysis-asr-extensions/tasks.md‎
Lines changed: 251 additions & 0 deletions
diff --git a/‎src/senselab/audio/tasks/forced_alignment/constants.py‎
Lines changed: 48 additions & 0 deletions b/‎src/senselab/audio/tasks/forced_alignment/constants.py‎
Lines changed: 48 additions & 0 deletions
diff --git a/‎src/senselab/audio/tasks/forced_alignment/forced_alignment.py‎
Lines changed: 113 additions & 12 deletions b/‎src/senselab/audio/tasks/forced_alignment/forced_alignment.py‎
Lines changed: 113 additions & 12 deletions
diff --git a/‎src/senselab/audio/tasks/speech_to_text/api.py‎
Lines changed: 43 additions & 0 deletions b/‎src/senselab/audio/tasks/speech_to_text/api.py‎
Lines changed: 43 additions & 0 deletions
@@ -39,6 +39,7 @@ dependencies = [
   "transformers>=5.0",  # >=5.0 required for huggingface-hub>=1.0 compatibility
   "datasets>=3.6",
   "accelerate",
+  "peft>=0.13",  # required by Granite Speech 3.3's LoRA adapter
   "speechbrain>=1.0",
   # Audio processing backends (pyannote pulls matplotlib, torch-audiomentations)
   "pyannote-audio>=4.0",
@@ -59,7 +60,8 @@ dependencies = [
 [project.optional-dependencies]
 nlp = [
   "jiwer>=3.0",
-  "nltk>=3.9"
+  "nltk>=3.9",
+  "uroman>=1.3"
 ]
 text = [
   "sentence-transformers>=5.1",
@@ -196,4 +198,4 @@ skip = [
   "*.cha",
   "*.ipynb"
 ]
-ignore-words-list = ["senselab", "nd", "astroid", "wil", "SER", "te", "EXPRESSO", "VAI"]
+ignore-words-list = ["senselab", "nd", "astroid", "wil", "SER", "te", "EXPRESSO", "VAI", "vie"]
@@ -31,7 +31,7 @@ def main() -> None:
         for m in task_models:
             name = m["name"]
             source = m["source"]
-            model_id = f'`{m["model_id"]}`'
+            model_id = f"`{m['model_id']}`"
             emb = m.get("embedding_dim", "—")
             params = m.get("parameters", "—")
             rec = m.get("recommended_for", "—")
 
@@ -137,11 +137,7 @@ def _load_mono_16k():
     from senselab.audio.tasks.preprocessing import resample_audios
 
     mono_path = (
-        Path(__file__).resolve().parent.parent
-        / "src"
-        / "tests"
-        / "data_for_testing"
-        / "audio_48khz_mono_16bits.wav"
+        Path(__file__).resolve().parent.parent / "src" / "tests" / "data_for_testing" / "audio_48khz_mono_16bits.wav"
     )
     audio = Audio(filepath=str(mono_path))
     return resample_audios([audio], 16000)[0]
@@ -259,11 +255,7 @@ def setup_wav2vec2_emotion_dim(device: str):
     from senselab.utils.data_structures import DeviceType, HFModel
 
     mono_path = (
-        Path(__file__).resolve().parent.parent
-        / "src"
-        / "tests"
-        / "data_for_testing"
-        / "audio_48khz_mono_16bits.wav"
+        Path(__file__).resolve().parent.parent / "src" / "tests" / "data_for_testing" / "audio_48khz_mono_16bits.wav"
     )
     audio = Audio(filepath=str(mono_path))
     resampled = resample_audios([audio], 16000)
@@ -290,11 +282,7 @@ def setup_wav2vec2_xlsr_emotion(device: str):
     from senselab.utils.data_structures import DeviceType, HFModel
 
     mono_path = (
-        Path(__file__).resolve().parent.parent
-        / "src"
-        / "tests"
-        / "data_for_testing"
-        / "audio_48khz_mono_16bits.wav"
+        Path(__file__).resolve().parent.parent / "src" / "tests" / "data_for_testing" / "audio_48khz_mono_16bits.wav"
     )
     audio = Audio(filepath=str(mono_path))
     resampled = resample_audios([audio], 16000)
@@ -532,8 +520,7 @@ def print_summary_table(results: List[ProfileResult]) -> None:
     """Print a human-readable summary table."""
     print("\n" + "=" * 120)
     print(
-        f"{'Model':<55} {'Task':<22} {'Device':<6} {'Cold(s)':<8} {'Warm(s)':<8} "
-        f"{'RAM(MB)':<9} {'GPU(MB)':<9} {'Tier'}"
+        f"{'Model':<55} {'Task':<22} {'Device':<6} {'Cold(s)':<8} {'Warm(s)':<8} {'RAM(MB)':<9} {'GPU(MB)':<9} {'Tier'}"
     )
     print("-" * 120)
 
@@ -693,9 +680,7 @@ def pytest_sessionfinish(self, session, exitstatus):
 
         if torch.cuda.is_available():
             report["system"]["gpu_name"] = torch.cuda.get_device_name(0)
-            report["system"]["gpu_memory_gb"] = round(
-                torch.cuda.get_device_properties(0).total_mem / (1024**3), 1
-            )
+            report["system"]["gpu_memory_gb"] = round(torch.cuda.get_device_properties(0).total_mem / (1024**3), 1)
 
         # Summary stats
         times = [r["wall_seconds"] for r in self.results]
 
@@ -8,6 +8,54 @@
 
 LANGUAGES_WITHOUT_SPACES = ["ja", "zh"]
 
+# MMS (Massively Multilingual Speech) — single-model fallback aligner
+# covering 1100+ languages via per-language adapters. Selected via the
+# ``align_transcriptions(..., aligner_model=MMS_MODEL_ID)`` knob; not the
+# default for the languages already in DEFAULT_ALIGN_MODELS_HF /
+# DEFAULT_ALIGN_MODELS_TORCH (preserves backwards-compat for existing
+# callers, who keep getting the same per-language wav2vec2 models).
+#
+# Weights are CC-BY-NC 4.0 (research / non-commercial). Override via
+# the public API for commercial-friendly alternatives.
+MMS_MODEL_ID = "facebook/mms-1b-all"
+
+# ISO 639-1 -> ISO 639-3 map for MMS adapter selection. MMS adapters are
+# keyed by ISO-3 codes; senselab callers (and Language objects) typically
+# carry ISO-1. Languages absent from this map cannot be auto-resolved to
+# an MMS adapter and will raise an actionable error.
+ISO_1_TO_3 = {
+    "en": "eng",
+    "fr": "fra",
+    "de": "deu",
+    "es": "spa",
+    "it": "ita",
+    "pt": "por",
+    "ja": "jpn",
+    "zh": "cmn",  # Mandarin Chinese
+    "nl": "nld",
+    "uk": "ukr",
+    "ar": "ara",
+    "cs": "ces",
+    "ru": "rus",
+    "pl": "pol",
+    "hu": "hun",
+    "fi": "fin",
+    "fa": "fas",
+    "el": "ell",
+    "tr": "tur",
+    "da": "dan",
+    "he": "heb",
+    "vi": "vie",
+    "ko": "kor",
+    "ur": "urd",
+    "te": "tel",
+    "hi": "hin",
+    "ca": "cat",
+    "ml": "mal",
+    "no": "nor",
+    "nn": "nno",
+}
+
 DEFAULT_ALIGN_MODELS_TORCH = {
     "fr": {"path_or_uri": "VOXPOPULI_ASR_BASE_10K_FR", "revision": "main"},
     "de": {"path_or_uri": "VOXPOPULI_ASR_BASE_10K_DE", "revision": "main"},
 
@@ -10,8 +10,10 @@
 from senselab.audio.data_structures import Audio
 from senselab.audio.tasks.forced_alignment.constants import (
     DEFAULT_ALIGN_MODELS_HF,
+    ISO_1_TO_3,
     LANGUAGES_WITHOUT_SPACES,
     MINIMUM_SEGMENT_SIZE,
+    MMS_MODEL_ID,
     PUNKT_ABBREVIATIONS,
     SAMPLE_RATE,
 )
@@ -566,9 +568,72 @@ def filter_aligned_script_lines(
     return [flatten_script_lines(scriptline_list) for scriptline_list in aligned_script_lines]
 
 
+def _romanize_for_mms(text: str) -> str:
+    """Romanize a transcript so MMS's Roman-script adapter can tokenize it.
+
+    MMS uses Roman characters internally; for languages written in non-Roman
+    scripts (notably Japanese kanji/kana and Chinese Hanzi) the transcript
+    must be uroman-romanized before alignment. ``uroman`` is an optional
+    dependency in the ``[nlp]`` extra and is imported lazily so default
+    senselab installs are not affected.
+
+    Raises ``ImportError`` with an actionable hint when uroman is absent.
+    """
+    try:
+        from uroman import Uroman  # type: ignore[import-not-found]
+    except ImportError as exc:
+        raise ImportError(
+            "uroman is required to align ja/zh transcripts via MMS. "
+            "Install with `uv sync --extra nlp` (adds uroman to the venv) "
+            "or pass an aligner that does not require romanization."
+        ) from exc
+    uroman = Uroman()
+    return str(uroman.romanize_string(text))
+
+
+def _load_mms_aligner(
+    iso3: str,
+    device: DeviceType,
+    cache: Dict[Any, Any],
+) -> Tuple[Any, Any]:
+    """Load (or fetch from cache) the MMS Wav2Vec2 model + processor for one language adapter.
+
+    MMS-1B-All is a single base model with one set of weights and 1100+
+    swappable per-language adapters. We cache by (model_id, iso3) so two
+    languages on the same audio reuse the base weights but swap adapters.
+
+    Return types are intentionally ``Any`` because the transformers type
+    stubs do not expose ``Wav2Vec2Processor.tokenizer`` (PreTrainedTokenizer
+    on a multi-modal processor is duck-typed), and over-annotating triggers
+    spurious mypy errors. Runtime types are
+    ``(Wav2Vec2Processor, Wav2Vec2ForCTC)`` as documented.
+    """
+    key = (MMS_MODEL_ID, iso3)
+    if key not in cache:
+        processor = Wav2Vec2Processor.from_pretrained(MMS_MODEL_ID)
+        processor.tokenizer.set_target_lang(iso3)  # type: ignore[attr-defined]
+        model = Wav2Vec2ForCTC.from_pretrained(
+            MMS_MODEL_ID,
+            target_lang=iso3,
+            ignore_mismatched_sizes=True,
+        ).to(device.value)  # type: ignore[arg-type]
+        model.load_adapter(iso3)
+        cache[key] = (processor, model)
+    return cache[key]
+
+
+# Module-level processor/model cache. Holds either ``(model_id, iso3) →
+# (processor, model)`` for the MMS path or ``model_id → (processor,
+# model)`` for the per-language wav2vec2 dispatch. Persists across
+# ``align_transcriptions`` calls so the 1B-parameter MMS weights load
+# once per process, not once per pass / audio.
+_ALIGNER_CACHE: Dict[Any, Any] = {}
+
+
 def align_transcriptions(
     audios_and_transcriptions_and_language: List[Tuple[Audio, ScriptLine, Language]],
     levels_to_keep: Dict = {"utterance": False, "word": False, "char": False},
+    aligner_model: Optional[str] = None,
 ) -> List[List[ScriptLine | None]]:
     """Align multiple transcriptions with their respective audios using a wav2vec2.0 model.
 
@@ -577,36 +642,70 @@ def align_transcriptions(
             Each tuple contains an Audio object, a ScriptLine with transcription,
             and an optional Language (default is English).
         levels_to_keep (Dict): Levels of transcription to keep in output.
+        aligner_model (str | None, optional):
+            Override the default per-language wav2vec2 aligner. When set to
+            ``MMS_MODEL_ID`` (``facebook/mms-1b-all``), the multilingual MMS
+            backend is used: a single model with per-language adapters
+            covering 1100+ languages. Adapter selection is driven by the
+            language code; ja/zh transcripts are romanized via uroman
+            (optional dep, ``[nlp]`` extra) before tokenization. When
+            ``None`` (default), the per-language wav2vec2 dispatch table
+            ``DEFAULT_ALIGN_MODELS_HF`` is used as before.
 
     Returns:
         List[List[ScriptLine | None]]: A list of aligned results for each audio.
     """
     aligned_script_lines: list[list[ScriptLine | None]] = []
-    loaded_processors_and_models = {}
+    # Module-level cache: each call into align_transcriptions reuses the
+    # 1B-parameter MMS base weights (and any wav2vec2 variants) from a
+    # prior call rather than reloading from disk every time.
+    loaded_processors_and_models: Dict[Any, Any] = _ALIGNER_CACHE
     device = _select_device_and_dtype()[0]
+    use_mms = aligner_model == MMS_MODEL_ID
 
     for recording in audios_and_transcriptions_and_language:
         audio, transcription, language = (*recording, None)[:3]
         if language is None:
             language = Language(language_code="en")
 
-        model_dict = DEFAULT_ALIGN_MODELS_HF.get(language.language_code, DEFAULT_ALIGN_MODELS_HF["en"])
-        model_variant: HFModel = HFModel(path_or_uri=model_dict["path_or_uri"], revision=model_dict["revision"])
-        if model_variant.path_or_uri not in loaded_processors_and_models:
-            processor = Wav2Vec2Processor.from_pretrained(model_variant.path_or_uri, revision=model_variant.revision)
-            model = Wav2Vec2ForCTC.from_pretrained(model_variant.path_or_uri, revision=model_variant.revision).to(
-                device.value
-            )
-            loaded_processors_and_models[model_variant.path_or_uri] = (processor, model)
-
-        processor, model = loaded_processors_and_models[model_variant.path_or_uri]
+        if use_mms:
+            code = language.language_code
+            # senselab's Language normalizes ISO-639-1 ("en") to ISO-639-3 ("eng").
+            # Accept either form: pass through 3-letter codes; map 2-letter codes
+            # via ISO_1_TO_3.
+            if len(code) == 3:
+                iso3: Optional[str] = code
+            else:
+                iso3 = ISO_1_TO_3.get(code)
+            if iso3 is None:
+                raise ValueError(
+                    f"MMS aligner has no ISO-639-3 mapping for language "
+                    f"{code!r}. Add it to ISO_1_TO_3 in "
+                    f"senselab.audio.tasks.forced_alignment.constants, or pass "
+                    f"a different aligner_model."
+                )
+            processor, model = _load_mms_aligner(iso3, device, loaded_processors_and_models)
+        else:
+            model_dict = DEFAULT_ALIGN_MODELS_HF.get(language.language_code, DEFAULT_ALIGN_MODELS_HF["en"])
+            model_variant: HFModel = HFModel(path_or_uri=model_dict["path_or_uri"], revision=model_dict["revision"])
+            if model_variant.path_or_uri not in loaded_processors_and_models:
+                processor = Wav2Vec2Processor.from_pretrained(
+                    model_variant.path_or_uri, revision=model_variant.revision
+                )
+                _w2v = Wav2Vec2ForCTC.from_pretrained(model_variant.path_or_uri, revision=model_variant.revision)
+                model = _w2v.to(device.value)  # type: ignore[arg-type]
+                loaded_processors_and_models[model_variant.path_or_uri] = (processor, model)
+            processor, model = loaded_processors_and_models[model_variant.path_or_uri]
 
         if audio.sampling_rate != SAMPLE_RATE:
             raise ValueError(f"{audio.sampling_rate} rate is not equal to {SAMPLE_RATE}.")
 
         start = transcription.start if transcription.start is not None else 0.0
         end = transcription.end if transcription.end is not None else audio.waveform.shape[1] / audio.sampling_rate
         text = transcription.text if transcription.text is not None else ""
+        # MMS uses Roman characters; ja/zh transcripts must be romanized first.
+        if use_mms and language.language_code in LANGUAGES_WITHOUT_SPACES:
+            text = _romanize_for_mms(text)
 
         segments = [
             SingleSegment(
@@ -621,7 +720,9 @@ def align_transcriptions(
                 align_model_metadata={
                     "dictionary": processor.tokenizer.get_vocab(),
                     "language": language,
-                    "type": "huggingface" if isinstance(model_variant, HFModel) else "torchaudio",
+                    # MMS shares the wav2vec2 forward pass with the per-language
+                    # HF aligners, so the trellis/backtrack code is identical.
+                    "type": "huggingface",
                 },
                 audio=audio,
                 device=device,
 
@@ -8,14 +8,41 @@
 from typing import Any, List, Optional
 
 from senselab.audio.data_structures import Audio
+from senselab.audio.tasks.speech_to_text.canary_qwen import CanaryQwenASR
+from senselab.audio.tasks.speech_to_text.granite import GraniteSpeechASR
 from senselab.audio.tasks.speech_to_text.huggingface import HuggingFaceASR
 from senselab.audio.tasks.speech_to_text.nemo import NeMoASR
+from senselab.audio.tasks.speech_to_text.qwen import QwenASR
 from senselab.utils.compatibility import requires_compatibility
 from senselab.utils.data_structures import DeviceType, HFModel, Language, ScriptLine, SenselabModel
 
 # NeMo model ID prefixes — route to NeMo backend when detected
 _NEMO_PREFIXES = ("nvidia/stt_", "nvidia/conformer")
 
+# NVIDIA Canary-Qwen prefix — route to a separate NeMo subprocess venv
+# (canary_qwen.py) that loads SALM from nemo.collections.speechlm2.models.
+# The implementation lives in a follow-up commit; this constant is used by
+# the dispatch table below so the routing structure is settled.
+_CANARY_PREFIXES = ("nvidia/canary-",)
+
+# Alibaba Qwen3-ASR prefix — route to a separate Qwen subprocess venv
+# (qwen.py) that uses Alibaba's qwen-asr Python wrapper plus the optional
+# Qwen3-ForcedAligner companion for word-level timestamps.
+_QWEN_ASR_PREFIXES = ("Qwen/Qwen3-ASR",)
+
+# IBM Granite Speech prefix — route to a separate granite subprocess venv
+# (granite.py). Granite uses GraniteSpeechProcessor's (text, audio, device)
+# signature, not the standard HF ASR pipeline contract, so it cannot be
+# driven through HuggingFaceASR. Text-only output; auto-aligned downstream.
+_GRANITE_PREFIXES = ("ibm-granite/granite-speech-",)
+
+# HuggingFace ASR models that are known to NOT produce native timestamps —
+# their HF pipelines reject return_timestamps. For these we default to
+# return_timestamps=False so the pipeline returns text-only ScriptLines;
+# downstream code (e.g., scripts/analyze_audio.py) can post-align via
+# senselab.audio.tasks.forced_alignment to add per-segment timestamps.
+_TIMESTAMP_LESS_HF_MODELS: tuple[str, ...] = ()
+
 
 @requires_compatibility("audio.tasks.speech_to_text.transcribe_audios")
 def transcribe_audios(
@@ -98,7 +125,23 @@ def transcribe_audios(
     try:
         if isinstance(model, HFModel) and str(model.path_or_uri).startswith(_NEMO_PREFIXES):
             return NeMoASR.transcribe_with_nemo(audios=audios, model=model, device=device)
+        elif isinstance(model, HFModel) and str(model.path_or_uri).startswith(_CANARY_PREFIXES):
+            return CanaryQwenASR.transcribe_with_canary_qwen(audios=audios, model=model, device=device)
+        elif isinstance(model, HFModel) and str(model.path_or_uri).startswith(_QWEN_ASR_PREFIXES):
+            qwen_kwargs: dict[str, Any] = {}
+            if "return_timestamps" in kwargs:
+                qwen_kwargs["return_timestamps"] = bool(kwargs.pop("return_timestamps"))
+            if "forced_aligner" in kwargs:
+                qwen_kwargs["forced_aligner"] = kwargs.pop("forced_aligner")
+            return QwenASR.transcribe_with_qwen(audios=audios, model=model, device=device, **qwen_kwargs)
+        elif isinstance(model, HFModel) and str(model.path_or_uri).startswith(_GRANITE_PREFIXES):
+            return GraniteSpeechASR.transcribe_with_granite(audios=audios, model=model, device=device)
         elif isinstance(model, HFModel):
+            # Default HF pipeline path. Models known to lack native timestamps
+            # default to return_timestamps=False so the pipeline does not raise;
+            # callers can override by passing return_timestamps explicitly.
+            if "return_timestamps" not in kwargs and str(model.path_or_uri).startswith(_TIMESTAMP_LESS_HF_MODELS):
+                kwargs["return_timestamps"] = False
             return HuggingFaceASR.transcribe_audios_with_transformers(
                 audios=audios, model=model, language=language, device=device, **kwargs
             )