[fix] Fixes Qwen3-VL prompt expansion for multiple videos (#10518)

Co-authored-by: gemini-code-assist <200291788+gemini-code-assist@users.noreply.github.com>

Author: luca-888

src/llamafactory/data/mm_plugin.py

@@ -2379,16 +2379,13 @@ def process_messages(
 
             image_grid_thw = mm_inputs.get("image_grid_thw", [])
             video_grid_thw = mm_inputs.get("video_grid_thw", [])
-            num_frames = video_grid_thw[0][0] if len(video_grid_thw) > 0 else 0  # hard code for now
             video_metadata = mm_inputs.get("video_metadata", [])
 
         else:
             image_grid_thw = [None] * len(images)
             video_grid_thw = [None] * len(videos)
-            num_frames = 0
-            timestamps = [0]
 
-        for idx, message in enumerate(messages):
+        for message in messages:
             content = message["content"]
             while IMAGE_PLACEHOLDER in content:
                 image_seqlen = (
@@ -2403,19 +2400,17 @@ def process_messages(
 
             while VIDEO_PLACEHOLDER in content:
                 if self.expand_mm_tokens:
-                    metadata = video_metadata[idx]
+                    video_grid = video_grid_thw[num_video_tokens]
+                    num_frames = int(video_grid[0].item())
+                    metadata = video_metadata[num_video_tokens]
                     timestamps = processor._calculate_timestamps(
                         metadata.frames_indices,
                         metadata.fps,
-                        video_processor.merge_size,
+                        getattr(video_processor, "temporal_patch_size", 2),
                     )
                     video_structure = ""
+                    video_seqlen = int((video_grid[1:].prod() // video_merge_length).item())
                     for frame_index in range(num_frames):
-                        video_seqlen = (
-                            video_grid_thw[num_video_tokens][1:].prod() // video_merge_length
-                            if self.expand_mm_tokens
-                            else 1
-                        )
                         timestamp_sec = timestamps[frame_index]
                         frame_structure = (
                             f"<{timestamp_sec:.1f} seconds>"

tests/data/test_mm_plugin.py

@@ -424,19 +424,31 @@ def test_qwen3_vl_plugin():
     tokenizer_module = _load_tokenizer_module(model_name_or_path="Qwen/Qwen3-VL-30B-A3B-Instruct")
     qwen3_vl_plugin = get_mm_plugin(name="qwen3_vl", video_token="<|video_pad|>")
     check_inputs = {"plugin": qwen3_vl_plugin, **tokenizer_module}
+    video_token = "<|video_pad|>" * frame_seqlen
+    first_video = (
+        f"<0.2 seconds><|vision_start|>{video_token}<|vision_end|>"
+        f"<1.2 seconds><|vision_start|>{video_token}<|vision_end|>"
+    )
+    second_video = first_video + f"<2.2 seconds><|vision_start|>{video_token}<|vision_end|>"
+    videos = [
+        [Image.new("RGB", (32, 32), (255, 255, 255))] * 4,
+        [Image.new("RGB", (32, 32), (255, 255, 255))] * 6,
+    ]
+    messages = [
+        {"role": "user", "content": "Compare these videos: <video> and <video>."},
+        {"role": "assistant", "content": "They are different."},
+    ]
     check_inputs["expected_mm_messages"] = [
-        {
-            key: value.replace(
-                "<video>",  # little different with original processor for default `fps=2` in our repo
-                "<0.2 seconds><|vision_start|>{}<|vision_end|><1.2 seconds><|vision_start|>{}<|vision_end|>".format(
-                    "<|video_pad|>" * frame_seqlen, "<|video_pad|>" * frame_seqlen
-                ),
-            )
-            for key, value in message.items()
-        }
-        for message in VIDEO_MESSAGES
+        {key: value.replace("<video>", first_video) for key, value in message.items()} for message in VIDEO_MESSAGES
     ]
     _check_plugin(**check_inputs)
+    assert qwen3_vl_plugin.process_messages(messages, NO_IMAGES, videos, NO_AUDIOS, tokenizer_module["processor"]) == [
+        {
+            "role": "user",
+            "content": f"Compare these videos: {first_video} and {second_video}.",
+        },
+        {"role": "assistant", "content": "They are different."},
+    ]
 
 
 @pytest.mark.runs_on(["cpu", "mps"])