Audio from video in jinja

src/transformers/processing_utils.py

@@ -2103,18 +2103,15 @@ def apply_chat_template(
 
         # Set the sampling rate to load the audio files if user hasn't already passed with `kwargs`.
         audio_kwargs_from_user = processor_kwargs.get("audio_kwargs", {})
-        sampling_rate = kwargs.get(
-            "sampling_rate", processor_kwargs.get("sampling_rate", audio_kwargs_from_user.get("sampling_rate"))
-        )
+        sampling_rate = processor_kwargs.get("sampling_rate", audio_kwargs_from_user.get("sampling_rate"))
         if sampling_rate is None:
             if hasattr(self._audio_processor, "sampling_rate"):
                 sampling_rate = self._audio_processor.sampling_rate
             else:
                 sampling_rate = 16_000
 
-        load_audio_backend = kwargs.get(
-            "load_audio_backend",
-            processor_kwargs.get("load_audio_backend", audio_kwargs_from_user.get("load_audio_backend")),
+        load_audio_backend = processor_kwargs.get(
+            "load_audio_backend", audio_kwargs_from_user.get("load_audio_backend")
         )
         if load_audio_backend is None:
             default_audio_kwargs = self.valid_processor_kwargs._defaults.get("audio_kwargs", {})
@@ -2187,9 +2184,13 @@ def apply_chat_template(
                             )
                     else:
                         for fname in video_fnames:
-                            # This updates the template in-place and adds audio entry
-                            # to ensure `audio` token is added by jinja
-                            message["content"].append({"type": "audio"})
+                            # This updates the template in-place and adds audio entry to ensure `audio` token is added by jinja
+                            if {"type": "audio"} not in message["content"]:
+                                message["content"].append({"type": "audio"})
+                                logger.warning(
+                                    "When setting `load_audio_from_video=True` you must add an empty `audio` entry in the conversation. "
+                                    "From v5.25 the entry will NOT be added automatically during when calling `processor.apply_chat_template()`"
+                                )
                             batch_audios.append(
                                 load_audio(fname, sampling_rate=sampling_rate, backend=load_audio_backend)
                             )

tests/test_processing_common.py

@@ -24,7 +24,6 @@
 from unittest.mock import patch
 
 import numpy as np
-from huggingface_hub import hf_hub_download
 from parameterized import parameterized
 
 from transformers import ProcessorMixin
@@ -1483,34 +1482,30 @@ def test_apply_chat_template_video_frame_sampling(self):
                 processor_kwargs={"do_sample_frames": True},
             )
 
-    @require_librosa
     @require_torchcodec
     def test_chat_template_audio_from_video(self):
         processor = self.get_processor()
         if processor.chat_template is None:
             self.skipTest("Processor has no chat template")
 
-        signature = inspect.signature(processor.__call__)
-        if "videos" not in {*signature.parameters.keys()} or (
-            signature.parameters.get("videos") is not None
-            and signature.parameters["videos"].annotation == inspect._empty
-        ):
-            self.skipTest(f"{self.processor_class} does not support video inputs")
+        if "video_processor" not in self.processor_class.get_attributes():
+            self.skipTest(f"{self.processor_class.__name__} does not support video inputs")
 
         if (
             "feature_extractor" not in self.processor_class.get_attributes()
-            or "audio_processor" not in self.processor_class.get_attributes()
+            and "audio_processor" not in self.processor_class.get_attributes()
         ):
-            self.skipTest(f"feature_extractor attribute not present in {self.processor_class}")
+            self.skipTest(f"{self.processor_class.__name__} does not support audio inputs")
 
-        video_file_path = hf_hub_download(
-            repo_id="hf-internal-testing/test-videos", filename="sample_demo_1_320x240.mp4", repo_type="dataset"
+        video_file_path = url_to_local_path(
+            "https://huggingface.co/datasets/hf-internal-testing/test-videos/resolve/main/sample_demo_1_320x240.mp4"
         )
         messages = [
             {
                 "role": "user",
                 "content": [
                     {"type": "video", "path": video_file_path},
+                    {"type": "audio"},
                     {"type": "text", "text": "Which of these animals is making the sound?"},
                 ],
             },
@@ -1535,16 +1530,15 @@ def test_chat_template_audio_from_video(self):
             tokenize=True,
             return_dict=True,
             return_tensors="pt",
+            load_audio_backend="torchcodec",
             load_audio_from_video=True,
         )
         self.assertTrue(self.audio_input_name in out_dict)
         self.assertTrue(self.videos_input_name in out_dict)
 
-        # should always have input_ids and attention_mask
-        self.assertEqual(len(out_dict["input_ids"]), 1)  # batch-size=1
-        self.assertEqual(len(out_dict["attention_mask"]), 1)  # batch-size=1
-        self.assertEqual(len(out_dict[self.audio_input_name]), 1)  # 1 audio in the conversation
-        self.assertEqual(len(out_dict[self.videos_input_name]), 1)  # 1 video in the conversation
+        self.assertEqual(out_dict["input_ids"].shape[0], 1)  # batch-size=1
+        self.assertEqual(out_dict["attention_mask"].shape[0], 1)  # batch-size=1
+        self.assertEqual(out_dict[self.audio_input_name].shape[0], 1)  # 1 audio from video
 
     def test_chat_template_jinja_kwargs(self):
         """Tests that users can pass any kwargs and they will be used in jinja templates."""