Add YAMNet subprocess backend and auto-resampling for classification (#505)

* Add AST + YAMNet to classification docs and model registry

- classification/doc.md: AST and YAMNet added to Models section
  (organized by category: scene, demographics, content)
- model_registry.yaml: new audio_scene_classification section
- model_registry.md: regenerated
- Fixed generate_model_registry.py path (docs/ → src/senselab/)

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* Add __all__ to all task/workflow modules for clean pdoc docs

Without __all__, pdoc shows internal submodules (api, huggingface,
etc.) as navigation items. With __all__, pdoc only shows the public
API functions — what users actually need.

Applied to 21 task modules + 1 workflow module across audio, video,
text, and utils.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* Fix review: YAMNet TF note, music genre link, model registry source

- YAMNet: noted as TensorFlow-based, not directly supported via classify_audios
- Music genre model link: config.json → main model card
- Model registry: YAMNet source changed from huggingface to tensorflow

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* Add YAMNet subprocess venv backend and auto-resampling for classification

- YAMNet runs in isolated TF subprocess venv (tensorflow, tensorflow-hub)
- Accessible via classify_audios(audios, model="yamnet")
- Auto-resamples to 16kHz inside the loop (memory-efficient)
- HF classifier now auto-resamples instead of erroring on wrong sample rate
- Updated doc.md and model_registry to reflect YAMNet as fully supported

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* Fix mypy: cast model.path_or_uri to str for hf_hub_download

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* Apply ruff format to yamnet.py

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: satra

scripts/generate_model_registry.py

@@ -8,7 +8,7 @@
 
 def main() -> None:
     """Read YAML and output Markdown table."""
-    registry_path = Path(__file__).parent.parent / "docs" / "model_registry.yaml"
+    registry_path = Path(__file__).parent.parent / "src" / "senselab" / "model_registry.yaml"
     with open(registry_path) as f:
         models = yaml.safe_load(f)
 

src/senselab/audio/tasks/classification/__init__.py

@@ -1,3 +1,7 @@
 """.. include:: ./doc.md"""  # noqa: D415
 
 from .api import classify_audios, scene_results_to_segments  # noqa: F401
+from .speech_emotion_recognition import classify_emotions_from_speech  # noqa: F401
+from .yamnet import YAMNetClassifier  # noqa: F401
+
+__all__ = ["classify_audios", "classify_emotions_from_speech", "scene_results_to_segments", "YAMNetClassifier"]

src/senselab/audio/tasks/classification/api.py

@@ -1,26 +1,36 @@
 """This module represents the API for the speech classification task within the senselab package.
 
-Currently, it supports only Hugging Face models, with plans to include more in the future.
-Users can specify the audio clips to classify, the classification model, the preferred device,
-and the model-specific parameters, and senselab handles the rest.
+Supports HuggingFace models and YAMNet (via isolated subprocess venv).
+Users can specify the audio clips to classify, the classification model,
+the preferred device, and the model-specific parameters.
 
 When ``win_length`` is provided, classification runs over sliding windows,
 producing per-window results with timestamps. If omitted, the entire audio
-is classified as a single unit.
+is classified as a single unit (HF models) or using the model's own
+internal windowing (YAMNet).
 """
 
 from typing import Any, Dict, List, Optional, Union
 
 from senselab.audio.data_structures import Audio, AudioClassificationResult
 from senselab.audio.tasks.classification.huggingface import HuggingFaceAudioClassifier
 from senselab.utils.compatibility import requires_compatibility
-from senselab.utils.data_structures import DeviceType, HFModel, SenselabModel
+from senselab.utils.data_structures import DeviceType, HFModel, SenselabModel, logger
+
+_YAMNET_ALIASES = {"yamnet", "google/yamnet"}
+
+
+def _is_yamnet(model: Union[SenselabModel, str]) -> bool:
+    """Check if the model refers to YAMNet."""
+    if isinstance(model, str):
+        return model.lower() in _YAMNET_ALIASES
+    return False
 
 
 @requires_compatibility("audio.tasks.classification.classify_audios")
 def classify_audios(
     audios: List[Audio],
-    model: SenselabModel,
+    model: Union[SenselabModel, str],
     device: Optional[DeviceType] = None,
     win_length: Optional[float] = None,
     hop_length: Optional[float] = None,
@@ -29,8 +39,9 @@ def classify_audios(
 ) -> Union[List[AudioClassificationResult], List[List[Dict[str, Any]]]]:
     """Classify audios using the given model.
 
-    When ``win_length`` is ``None`` (default), each audio is classified as a
-    whole and the return type is ``List[AudioClassificationResult]``.
+    When ``win_length`` is ``None`` (default) and the model is an HF model,
+    each audio is classified as a whole and the return type is
+    ``List[AudioClassificationResult]``.
 
     When ``win_length`` is provided (in seconds), each audio is sliced into
     overlapping windows and classified per-window.  The return type changes
@@ -42,23 +53,39 @@ def classify_audios(
     - ``win_length`` / ``hop_length`` (float): the parameters used
       (captured for provenance).
 
+    **YAMNet** (``model="yamnet"``): Runs in an isolated TensorFlow
+    subprocess venv.  YAMNet uses fixed 0.96 s windows with 0.48 s hop
+    internally, so it always returns windowed results.  The ``win_length``
+    and ``hop_length`` parameters are ignored for YAMNet.
+
     Args:
         audios: Audio objects to classify.
-        model: The classification model.
-        device: Device for inference (default: auto-select).
+        model: The classification model.  Can be an ``HFModel`` for
+            HuggingFace pipelines, or ``"yamnet"`` for the YAMNet
+            subprocess backend.
+        device: Device for inference (default: auto-select).  Ignored
+            for YAMNet (TensorFlow manages devices internally).
         win_length: Window duration in seconds.  If ``None``, classify
             the full audio.  If set, ``hop_length`` defaults to
-            ``win_length / 2`` when not provided.
+            ``win_length / 2`` when not provided.  Ignored for YAMNet.
         hop_length: Hop (step) duration in seconds for windowed mode.
-            Ignored when ``win_length`` is ``None``.
+            Ignored when ``win_length`` is ``None``.  Ignored for YAMNet.
         top_k: Keep only the top-k labels per result.  Applies in both
-            whole-audio and windowed modes.  ``None`` keeps all labels.
+            whole-audio and windowed modes.  ``None`` keeps all labels
+            (defaults to 5 for windowed mode and YAMNet).
         **kwargs: Forwarded to the backend classifier.
 
     Returns:
-        ``List[AudioClassificationResult]`` in whole-audio mode, or
-        ``List[List[Dict]]`` in windowed mode.
+        ``List[AudioClassificationResult]`` in whole-audio mode (HF only), or
+        ``List[List[Dict]]`` in windowed mode or when using YAMNet.
     """
+    if _is_yamnet(model):
+        from senselab.audio.tasks.classification.yamnet import YAMNetClassifier
+
+        if win_length is not None:
+            logger.info("YAMNet uses fixed 0.96s windows; win_length/hop_length parameters are ignored.")
+        return YAMNetClassifier.classify_with_yamnet(audios=audios, top_k=top_k or 5)
+
     if win_length is not None:
         return _classify_windowed(
             audios=audios,
@@ -79,7 +106,7 @@ def classify_audios(
 
 def _classify_whole(
     audios: List[Audio],
-    model: SenselabModel,
+    model: Union[SenselabModel, str],
     device: Optional[DeviceType] = None,
     **kwargs: Any,  # noqa: ANN401
 ) -> List[AudioClassificationResult]:
@@ -89,13 +116,14 @@ def _classify_whole(
             audios=audios, model=model, device=device, **kwargs
         )
     raise NotImplementedError(
-        "Only Hugging Face models are supported for now. We aim to support more models in the future."
+        f"Model type {type(model).__name__} is not supported for whole-audio classification. "
+        "Use HFModel for HuggingFace pipelines or 'yamnet' for YAMNet."
     )
 
 
 def _classify_windowed(
     audios: List[Audio],
-    model: SenselabModel,
+    model: Union[SenselabModel, str],
     device: Optional[DeviceType],
     win_length: float,
     hop_length: float,

src/senselab/audio/tasks/classification/doc.md

@@ -9,10 +9,18 @@ Audio Classification refers to the broad category of tasks of processing audio i
 A variety of models are supported by ```senselab``` for audio classification tasks. They can be explored on the [Hugging Face Hub](https://huggingface.co/models?library=transformers&pipeline_tag=audio-classification&sort=downloads). Each model varies in performance, size, license, language support, and more. Like with many models and tasks, performance may also vary depending on who the speaker is in the processed audio clips (there may be differences in terms of age, dialects, disfluencies). It is recommended to review the model card for each model before use. Also, always refer to the most recent literature for an informed decision. Unlike other tasks, the exact classification task will be based off of the dataset and class labels used to train the model, such that even two models using the same dataset might classify the audios into different categories (e.g. the LibriSpeech dataset could be used to classify age and/or gender based on the audio clips).
 
 Some popular models for different classifications include:
+
+**Auditory Scene / Sound Event Classification:**
+  - [Audio Spectrogram Transformer (AST)](https://huggingface.co/MIT/ast-finetuned-audioset-10-10-0.4593) — 521 AudioSet classes (speech, music, environmental sounds, animals, etc.). Recommended for general-purpose scene analysis.
+  - [YAMNet](https://tfhub.dev/google/yamnet/1) — Google's AudioSet classifier (521 classes). TensorFlow-based; runs in an isolated subprocess venv via `classify_audios(audios, model="yamnet")`.
+
+**Speaker / Demographics:**
   - [Age](https://huggingface.co/audeering/wav2vec2-large-robust-24-ft-age-gender)
   - [Gender](https://huggingface.co/alefiury/wav2vec2-large-xlsr-53-gender-recognition-librispeech)
-  - [Music Genre](https://huggingface.co/agercas/distilhubert-finetuned-gtzan/blob/main/config.json)
   - [Adult vs. Child Speech](https://huggingface.co/bookbot/wav2vec2-adult-child-cls)
+
+**Content:**
+  - [Music Genre](https://huggingface.co/agercas/distilhubert-finetuned-gtzan)
   - [Emotion Recognition](https://huggingface.co/ehcalabres/wav2vec2-lg-xlsr-en-speech-emotion-recognition)
 
 ## Evaluation

src/senselab/audio/tasks/classification/huggingface.py

@@ -11,6 +11,7 @@
 from transformers import Pipeline, pipeline
 
 from senselab.audio.data_structures import Audio, AudioClassificationResult
+from senselab.audio.tasks.preprocessing import resample_audios
 from senselab.utils.data_structures import DeviceType, HFModel, _select_device_and_dtype
 from senselab.utils.data_structures.logging import logger
 
@@ -131,17 +132,15 @@ def _audio_to_huggingface_dict(audio: Audio) -> Dict:
         if expected_sampling_rate is None:
             raise ValueError("Internal error: The Hugging Face model does not specify an expected sampling rate.")
 
-        # Check that all audio objects are mono
+        # Validate mono and resample to expected rate inside the loop
+        # to avoid holding all resampled audios in memory at once
+        formatted_audios = []
         for audio in audios:
             if audio.waveform.shape[0] != 1:
                 raise ValueError(f"Stereo audio is not supported. Got {audio.waveform.shape[0]} channels")
             if audio.sampling_rate != expected_sampling_rate:
-                raise ValueError(
-                    f"Incorrect sampling rate. Expected {expected_sampling_rate}, got {audio.sampling_rate}"
-                )
-
-        # Convert the audio objects to dictionaries that can be used by the pipeline
-        formatted_audios = [_audio_to_huggingface_dict(audio) for audio in audios]
+                audio = resample_audios([audio], resample_rate=expected_sampling_rate)[0]
+            formatted_audios.append(_audio_to_huggingface_dict(audio))
 
         # Take the start time of the transcription
         start_time_transcription = time.time()

src/senselab/audio/tasks/classification/speech_emotion_recognition/__init__.py

@@ -1,3 +1,5 @@
 """.. include:: ./doc.md"""  # noqa: D415
 
 from .api import classify_emotions_from_speech  # noqa: F401
+
+__all__ = ["classify_emotions_from_speech"]

src/senselab/audio/tasks/classification/speech_emotion_recognition/api.py

@@ -210,7 +210,7 @@ def _get_ser_type(model: HFModel) -> SERType:
         # Fall back to raw config dict for models with invalid fields
         from huggingface_hub import hf_hub_download
 
-        config_path = hf_hub_download(model.path_or_uri, "config.json", revision=model.revision)
+        config_path = hf_hub_download(str(model.path_or_uri), "config.json", revision=model.revision)
         with open(config_path) as f:
             config_dict = json.load(f)
         config = SimpleNamespace(id2label=config_dict.get("id2label", {}))

src/senselab/audio/tasks/classification/yamnet.py

@@ -0,0 +1,160 @@
+"""YAMNet audio classification via isolated subprocess venv.
+
+YAMNet is a TensorFlow-based model that classifies audio into 521
+AudioSet classes. It runs in an isolated subprocess venv to avoid
+TF/PyTorch conflicts.
+"""
+
+import json
+import subprocess
+import tempfile
+from pathlib import Path
+from typing import Any, Dict, List
+
+from senselab.audio.data_structures import Audio
+from senselab.audio.tasks.preprocessing import resample_audios
+from senselab.utils.subprocess_venv import _clean_subprocess_env, ensure_venv, parse_subprocess_result
+
+_YAMNET_VENV = "yamnet"
+_YAMNET_REQUIREMENTS = [
+    "tensorflow",
+    "tensorflow-hub",
+    "setuptools<70",  # tensorflow-hub needs pkg_resources
+    "numpy",
+    "soundfile",
+]
+_YAMNET_PYTHON = "3.12"
+
+_YAMNET_WORKER = r"""
+import json
+import sys
+
+try:
+    import numpy as np
+    import soundfile as sf
+    import tensorflow_hub as hub
+
+    args = json.loads(sys.stdin.read())
+    audio_paths = args["audio_paths"]
+    top_k = args.get("top_k", 5)
+
+    # Load YAMNet
+    model = hub.load("https://tfhub.dev/google/yamnet/1")
+
+    # Load class names from the model's assets
+    import csv
+    class_map_path = model.class_map_path().numpy().decode("utf-8")
+    with open(class_map_path) as f:
+        reader = csv.DictReader(f)
+        class_names = [row["display_name"] for row in reader]
+
+    all_results = []
+    for audio_path in audio_paths:
+        # Audio is already resampled to 16kHz mono by the caller
+        data, sr = sf.read(audio_path, dtype="float32")
+        if data.ndim > 1:
+            data = data.mean(axis=1)
+
+        scores, embeddings, spectrogram = model(data)
+        scores_np = scores.numpy()
+
+        # Each row in scores is a ~0.96s window
+        windows = []
+        for i, frame_scores in enumerate(scores_np):
+            top_indices = frame_scores.argsort()[::-1][:top_k]
+            windows.append({
+                "labels": [class_names[idx] for idx in top_indices],
+                "scores": [float(frame_scores[idx]) for idx in top_indices],
+            })
+        all_results.append(windows)
+
+    print(json.dumps({"results": all_results}))
+except Exception as exc:
+    print(json.dumps({"error": {"type": type(exc).__name__, "message": str(exc)}}))
+    sys.exit(1)
+"""
+
+
+class YAMNetClassifier:
+    """YAMNet audio classification via isolated subprocess venv."""
+
+    # YAMNet uses fixed 0.96s windows with 0.48s hop internally
+    WINDOW_SECONDS = 0.96
+    HOP_SECONDS = 0.48
+
+    @classmethod
+    def classify_with_yamnet(
+        cls,
+        audios: List[Audio],
+        top_k: int = 5,
+    ) -> List[List[Dict[str, Any]]]:
+        """Classify audios using YAMNet (521 AudioSet classes).
+
+        YAMNet uses its own internal windowing (0.96s windows, 0.48s hop).
+        Each audio produces multiple per-window results.
+
+        Args:
+            audios: Audio objects (mono, any sample rate — resampled to 16kHz internally).
+            top_k: Number of top labels per window.
+
+        Returns:
+            List of per-audio results, each containing per-window dicts
+            with ``labels``, ``scores``, ``start``, ``end``.
+        """
+        venv_dir = ensure_venv(_YAMNET_VENV, _YAMNET_REQUIREMENTS, python_version=_YAMNET_PYTHON)
+        python = str(venv_dir / "bin" / "python")
+
+        with tempfile.TemporaryDirectory(prefix="senselab-yamnet-") as tmpdir:
+            tmp = Path(tmpdir)
+
+            audio_paths = []
+            durations = []
+            for i, audio in enumerate(audios):
+                # Resample to 16kHz inside the loop to avoid holding all
+                # resampled audios in memory simultaneously
+                resampled = resample_audios([audio], resample_rate=16000)[0]
+                path = str(tmp / f"audio_{i}.wav")
+                resampled.save_to_file(path)
+                audio_paths.append(path)
+                durations.append(resampled.waveform.shape[1] / resampled.sampling_rate)
+
+            input_json = json.dumps(
+                {
+                    "audio_paths": audio_paths,
+                    "top_k": top_k,
+                }
+            )
+
+            env = _clean_subprocess_env()
+            result = subprocess.run(
+                [python, "-c", _YAMNET_WORKER],
+                input=input_json,
+                capture_output=True,
+                text=True,
+                timeout=600,
+                env=env,
+            )
+
+            output = parse_subprocess_result(result, "YAMNet")
+
+            # Add timestamps to each window based on YAMNet's fixed windowing
+            all_results: List[List[Dict[str, Any]]] = []
+            for audio_idx, windows in enumerate(output.get("results", [])):
+                duration = durations[audio_idx] if audio_idx < len(durations) else 0.0
+                timestamped = []
+                for i, w in enumerate(windows):
+                    start = i * cls.HOP_SECONDS
+                    end = min(start + cls.WINDOW_SECONDS, duration)
+                    timestamped.append(
+                        {
+                            "start": start,
+                            "end": end,
+                            "labels": w["labels"],
+                            "scores": w["scores"],
+                            "win_length": cls.WINDOW_SECONDS,
+                            "hop_length": cls.HOP_SECONDS,
+                        }
+                    )
+                all_results.append(timestamped)
+
+            return all_results

src/senselab/audio/tasks/data_augmentation/__init__.py

@@ -1,3 +1,5 @@
 """.. include:: ./doc.md"""  # noqa: D415
 
 from .api import augment_audios  # noqa: F401
+
+__all__ = ["augment_audios"]

src/senselab/audio/tasks/features_extraction/__init__.py

@@ -9,3 +9,13 @@
     to_frame_major_posteriorgram,
 )
 from .sparc import SparcFeatureExtractor  # noqa: F401
+
+__all__ = [
+    "extract_features_from_audios",
+    "extract_mean_phoneme_durations",
+    "extract_ppg_segments",
+    "extract_ppgs_from_audios",
+    "plot_ppg_phoneme_timeline",
+    "to_frame_major_posteriorgram",
+    "SparcFeatureExtractor",
+]