|
| 1 | +"""DiariZen diarization via isolated subprocess venv. |
| 2 | +
|
| 3 | +DiariZen (BUTSpeechFIT, code MIT / **weights CC BY-NC 4.0 — non-commercial |
| 4 | +only**) is a WavLM-Conformer EEND + clustering diarization toolkit built on |
| 5 | +top of a *forked* pyannote-audio. The default checkpoint |
| 6 | +(``BUT-FIT/diarizen-wavlm-large-s80-md``) uses VBx clustering |
| 7 | +(``config.toml``'s ``[clustering.args] method = "VBxClustering"``), a |
| 8 | +clustering algorithm that only exists in that fork |
| 9 | +(``diarizen/pyannote-audio`` on GitHub) and is absent from the ``pyannote.audio`` |
| 10 | +package published on PyPI — so this backend installs the fork, not the |
| 11 | +upstream package, from the same repo via pip's ``#subdirectory=`` VCS syntax. |
| 12 | +
|
| 13 | +The upstream ``diarizen`` and forked ``pyannote-audio`` packages declare no |
| 14 | +runtime dependencies at all in their packaging metadata (an unmaintained |
| 15 | +``setup.cfg`` for the fork; no ``[project.dependencies]`` for ``diarizen`` |
| 16 | +itself) — upstream's own install recipe relies on a separate |
| 17 | +``requirements.txt`` instead. This backend pins that same dependency set |
| 18 | +explicitly here rather than depending on a file fetched over the network at |
| 19 | +install time. |
| 20 | +
|
| 21 | +No ``transformers`` dependency: unlike MOSS or VibeVoice, DiariZen's WavLM |
| 22 | +encoder is its own vendored, torchaudio-style implementation |
| 23 | +(``diarizen.models.module.wav2vec2``), not ``transformers.WavLMModel``. |
| 24 | +
|
| 25 | +Neither the diarization checkpoint nor its internal embedding-model dependency |
| 26 | +(``pyannote/wespeaker-voxceleb-resnet34-LM``, downloaded automatically by |
| 27 | +``DiariZenPipeline.from_pretrained``) is gated on Hugging Face, so — unlike |
| 28 | +the Pyannote backend already in this package — no ``HF_TOKEN`` is required. |
| 29 | +
|
| 30 | +``DiariZenPipeline`` hardcodes its device selection internally (CUDA if |
| 31 | +available, else CPU) with no override hook, so the ``device`` argument here |
| 32 | +only validates the request isn't impossible before spawning the subprocess; |
| 33 | +it can't force a specific device inside the pipeline itself. |
| 34 | +""" |
| 35 | + |
| 36 | +from __future__ import annotations |
| 37 | + |
| 38 | +import json |
| 39 | +import subprocess |
| 40 | +import tempfile |
| 41 | +from pathlib import Path |
| 42 | +from typing import List, Optional |
| 43 | + |
| 44 | +from senselab.audio.data_structures import Audio |
| 45 | +from senselab.utils.data_structures import DeviceType, HFModel, ScriptLine, _select_device_and_dtype |
| 46 | +from senselab.utils.subprocess_venv import _clean_subprocess_env, ensure_venv, parse_subprocess_result, venv_python |
| 47 | + |
| 48 | +_DIARIZEN_VENV = "diarizen" |
| 49 | +_DIARIZEN_REQUIREMENTS = [ |
| 50 | + "diarizen @ git+https://github.com/BUTSpeechFIT/DiariZen.git", |
| 51 | + "pyannote-audio @ git+https://github.com/BUTSpeechFIT/DiariZen.git#subdirectory=pyannote-audio", |
| 52 | + "torch>=2.1,<2.9", |
| 53 | + "torchaudio>=2.1.1,<2.9", |
| 54 | + # The rest mirror pyannote-audio/requirements.txt in the DiariZen repo (that |
| 55 | + # fork's own packaging declares no install_requires, so these must be pinned |
| 56 | + # here explicitly rather than resolved transitively). |
| 57 | + "asteroid-filterbanks>=0.4", |
| 58 | + "einops>=0.6.0", |
| 59 | + "huggingface_hub>=0.13.0", |
| 60 | + "lightning>=2.0.1", |
| 61 | + "omegaconf>=2.1,<3.0", |
| 62 | + "pyannote.core==5.0.0", |
| 63 | + "pyannote.database==5.1.3", |
| 64 | + "pyannote.metrics==3.2.1", |
| 65 | + "pyannote.pipeline==3.0.1", |
| 66 | + "pytorch_metric_learning>=2.1.0", |
| 67 | + "rich>=12.0.0", |
| 68 | + "semver>=3.0.0", |
| 69 | + "soundfile>=0.12.1", |
| 70 | + "speechbrain>=0.5.14", |
| 71 | + "tensorboardX>=2.6", |
| 72 | + "torch_audiomentations>=0.11.0", |
| 73 | + "torchmetrics>=0.11.0", |
| 74 | + "scipy", |
| 75 | + "toml", |
| 76 | + "psutil", |
| 77 | + "accelerate", |
| 78 | + "numpy<2", |
| 79 | +] |
| 80 | +_DIARIZEN_PYTHON = "3.12" |
| 81 | +_DIARIZEN_DEFAULT_MODEL = "BUT-FIT/diarizen-wavlm-large-s80-md" |
| 82 | + |
| 83 | +# Worker script — runs inside the isolated venv. |
| 84 | +_WORKER_SCRIPT = r""" |
| 85 | +import json |
| 86 | +import sys |
| 87 | +
|
| 88 | +try: |
| 89 | + args = json.loads(sys.stdin.read()) |
| 90 | + audio_paths = args["audio_paths"] |
| 91 | + model_name = args["model_name"] |
| 92 | +
|
| 93 | + import torch |
| 94 | +
|
| 95 | + # The DiariZen checkpoint was saved under an older PyTorch/Lightning, whose |
| 96 | + # pickled metadata embeds custom objects (torch's own TorchVersion, |
| 97 | + # pyannote-audio's Specifications, ...) beyond plain tensors. PyTorch >=2.6 |
| 98 | + # defaults `torch.load` to `weights_only=True`, which rejects any global not |
| 99 | + # explicitly allowlisted, and pyannote-audio's own internal loading code |
| 100 | + # doesn't take a weights_only override we could pass through. Force |
| 101 | + # `weights_only=False` for this isolated subprocess venv only — reasonable |
| 102 | + # here since the checkpoint comes straight from the official BUT-FIT HF |
| 103 | + # repo for the exact backend being loaded, same trust level as this |
| 104 | + # package's own `trust_remote_code=True` usage elsewhere. |
| 105 | + _original_torch_load = torch.load |
| 106 | +
|
| 107 | + def _torch_load_full(*load_args, **load_kwargs): |
| 108 | + load_kwargs["weights_only"] = False |
| 109 | + return _original_torch_load(*load_args, **load_kwargs) |
| 110 | +
|
| 111 | + torch.load = _torch_load_full |
| 112 | +
|
| 113 | + from diarizen.pipelines.inference import DiariZenPipeline |
| 114 | +
|
| 115 | + pipeline = DiariZenPipeline.from_pretrained(model_name) |
| 116 | +
|
| 117 | + all_results = [] |
| 118 | + for audio_path in audio_paths: |
| 119 | + annotation = pipeline(audio_path) |
| 120 | + segments = [ |
| 121 | + {"speaker": str(speaker), "start": float(turn.start), "end": float(turn.end)} |
| 122 | + for turn, _, speaker in annotation.itertracks(yield_label=True) |
| 123 | + ] |
| 124 | + segments.sort(key=lambda s: s["start"]) |
| 125 | + all_results.append(segments) |
| 126 | +
|
| 127 | + print(json.dumps({"results": all_results})) |
| 128 | +except Exception as exc: |
| 129 | + import traceback |
| 130 | + err = { |
| 131 | + "type": type(exc).__name__, |
| 132 | + "message": str(exc), |
| 133 | + "traceback": traceback.format_exc(limit=5), |
| 134 | + } |
| 135 | + print(json.dumps({"error": err})) |
| 136 | + sys.exit(1) |
| 137 | +""" |
| 138 | + |
| 139 | + |
| 140 | +def diarize_audios_with_diarizen( |
| 141 | + audios: List[Audio], |
| 142 | + model: Optional[HFModel] = None, |
| 143 | + device: Optional[DeviceType] = None, |
| 144 | +) -> List[List[ScriptLine]]: |
| 145 | + """Diarize audios with **DiariZen**; returns per-speaker segments per audio. |
| 146 | +
|
| 147 | + DiariZen (``BUT-FIT/diarizen-wavlm-large-s80-md`` by default) is a |
| 148 | + WavLM-Conformer EEND segmentation model followed by VBx clustering, run |
| 149 | + via ``DiariZenPipeline`` (from the ``diarizen`` package and its forked |
| 150 | + ``pyannote-audio`` dependency, both installed in this backend's isolated |
| 151 | + venv). Diarization only — no transcription (unlike MOSS/VibeVoice), |
| 152 | + so returned `ScriptLine`s carry `speaker`/`start`/`end` but no `text`. |
| 153 | +
|
| 154 | + **License note**: the ``diarizen`` and forked ``pyannote-audio`` code is |
| 155 | + MIT, but the pretrained model weights are **CC BY-NC 4.0 — non-commercial |
| 156 | + use only**. |
| 157 | +
|
| 158 | + Args: |
| 159 | + audios (list[Audio]): |
| 160 | + Audio clips to diarize. |
| 161 | + model (HFModel | None): |
| 162 | + Defaults to ``HFModel(path_or_uri="BUT-FIT/diarizen-wavlm-large-s80-md")``. |
| 163 | + Other checkpoints from the same org (e.g. ``BUT-FIT/diarizen-wavlm-large-s80-md-v2``, |
| 164 | + ``BUT-FIT/diarizen-meeting-base``) also work. |
| 165 | + device (DeviceType | None): |
| 166 | + Preferred device. ``DiariZenPipeline`` selects its device |
| 167 | + internally (CUDA if available, else CPU) with no override hook; |
| 168 | + this argument only validates the request isn't impossible before |
| 169 | + spawning the subprocess. |
| 170 | +
|
| 171 | + Returns: |
| 172 | + list[list[ScriptLine]]: One list per input audio; each `ScriptLine` carries |
| 173 | + `speaker` (e.g. `"SPEAKER_00"`), `start`, and `end`. |
| 174 | +
|
| 175 | + Example: |
| 176 | + >>> from pathlib import Path |
| 177 | + >>> from senselab.audio.data_structures import Audio |
| 178 | + >>> from senselab.utils.data_structures import DeviceType |
| 179 | + >>> a1 = Audio(filepath=Path("sample1.wav").resolve()) |
| 180 | + >>> lines = diarize_audios_with_diarizen([a1], device=DeviceType.CPU) # doctest: +SKIP |
| 181 | + >>> len(lines) == 1 # doctest: +SKIP |
| 182 | + True |
| 183 | + """ |
| 184 | + if model is None: |
| 185 | + model = HFModel(path_or_uri=_DIARIZEN_DEFAULT_MODEL) |
| 186 | + |
| 187 | + _select_device_and_dtype(user_preference=device, compatible_devices=[DeviceType.CUDA, DeviceType.CPU]) |
| 188 | + |
| 189 | + venv_dir = ensure_venv(_DIARIZEN_VENV, _DIARIZEN_REQUIREMENTS, python_version=_DIARIZEN_PYTHON) |
| 190 | + python = venv_python(venv_dir) |
| 191 | + |
| 192 | + with tempfile.TemporaryDirectory(prefix="senselab-diarizen-") as tmpdir: |
| 193 | + tmp = Path(tmpdir) |
| 194 | + audio_paths = [] |
| 195 | + for i, audio in enumerate(audios): |
| 196 | + path = str(tmp / f"audio_{i}.wav") |
| 197 | + audio.save_to_file(path) |
| 198 | + audio_paths.append(path) |
| 199 | + |
| 200 | + input_json = json.dumps( |
| 201 | + { |
| 202 | + "audio_paths": audio_paths, |
| 203 | + "model_name": str(model.path_or_uri), |
| 204 | + } |
| 205 | + ) |
| 206 | + |
| 207 | + env = _clean_subprocess_env() |
| 208 | + result = subprocess.run( |
| 209 | + [python, "-c", _WORKER_SCRIPT], |
| 210 | + input=input_json, |
| 211 | + capture_output=True, |
| 212 | + text=True, |
| 213 | + timeout=1200, |
| 214 | + env=env, |
| 215 | + ) |
| 216 | + |
| 217 | + output = parse_subprocess_result(result, "DiariZen") |
| 218 | + |
| 219 | + results: List[List[ScriptLine]] = [] |
| 220 | + for segments in output.get("results", []): |
| 221 | + script_lines = [ |
| 222 | + ScriptLine( |
| 223 | + speaker=str(seg.get("speaker", "")), |
| 224 | + start=float(seg.get("start", 0.0)), |
| 225 | + end=float(seg.get("end", 0.0)), |
| 226 | + ) |
| 227 | + for seg in segments |
| 228 | + ] |
| 229 | + results.append(sorted(script_lines, key=lambda x: x.start or 0.0)) |
| 230 | + |
| 231 | + return results |
0 commit comments