fix: handle mid-stream sample rate/format changes in decode_audio

faster_whisper/audio.py

@@ -93,7 +93,20 @@ def _group_frames(frames, num_samples=None):
 
     for frame in frames:
         frame.pts = None  # Ignore timestamp check.
-        fifo.write(frame)
+
+        try:
+            fifo.write(frame)
+        except ValueError:
+            # The FIFO locks onto the format/layout/rate of the first frame
+            # written to it and rejects any later frame that differs (e.g.
+            # a stream whose sample rate or sample format changes partway
+            # through). Flush what we have and start a new FIFO for the
+            # frame with the new parameters; the downstream AudioResampler
+            # will normalize both groups to a common format anyway.
+            if fifo.samples > 0:
+                yield fifo.read()
+            fifo = av.audio.fifo.AudioFifo()
+            fifo.write(frame)
 
         if num_samples is not None and fifo.samples >= num_samples:
             yield fifo.read()
@@ -105,7 +118,23 @@ def _group_frames(frames, num_samples=None):
 def _resample_frames(frames, resampler):
     # Add None to flush the resampler.
     for frame in itertools.chain(frames, [None]):
-        yield from resampler.resample(frame)
+        try:
+            yield from resampler.resample(frame)
+        except ValueError:
+            # Like AudioFifo, AudioResampler locks onto the format/layout/
+            # rate of the first frame it processes and rejects later frames
+            # with different source parameters (e.g. a stream whose sample
+            # rate changes partway through). Flush the current resampler
+            # and swap in a fresh one (same target params) for the frame
+            # with the new source parameters.
+            yield from resampler.resample(None)
+            resampler = av.audio.resampler.AudioResampler(
+                format=resampler.format,
+                layout=resampler.layout,
+                rate=resampler.rate,
+            )
+            if frame is not None:
+                yield from resampler.resample(frame)
 
 
 def pad_or_trim(array, length: int = 3000, *, axis: int = -1):

tests/test_audio.py

@@ -0,0 +1,20 @@
+import os
+
+from faster_whisper.audio import decode_audio
+
+
+def test_decode_audio_with_mixed_sample_rate():
+    # Regression test for https://github.com/SYSTRAN/faster-whisper/issues/1451
+    #
+    # This file's audio stream changes sample rate partway through
+    # (44100 Hz -> 48000 Hz). Both AudioFifo and AudioResampler lock onto
+    # the parameters of the first frame they see and raise ValueError when
+    # a later frame doesn't match, so decode_audio used to crash on files
+    # like this one instead of decoding them.
+    path = os.path.join(os.path.dirname(__file__), "data", "mixed_samplerate.mp3")
+
+    audio = decode_audio(path)
+
+    assert audio.ndim == 1
+    assert audio.dtype.name == "float32"
+    assert audio.shape[0] > 0