diff --git a/tests/multimodal/test_processing.py b/tests/multimodal/test_processing.py index 4a41013b21a5..38e784d4bf18 100644 --- a/tests/multimodal/test_processing.py +++ b/tests/multimodal/test_processing.py @@ -11,7 +11,7 @@ from vllm.exceptions import VLLMValidationError from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.hasher import MultiModalHasher -from vllm.multimodal.parse import MultiModalDataParser +from vllm.multimodal.parse import AudioProcessorItems, MultiModalDataParser from vllm.multimodal.processing.context import ( InputProcessingContext, overlay_modality_mm_kwargs, @@ -1276,9 +1276,16 @@ def test_processor_inputs_hashes_partial_uuids(): mm_uuid_items={"image": ["image-uuid", None]}, ) + width, height = images[0].size assert inputs.get_mm_hashes("test-model", "blake3") == { "image": [ - "image-uuid", + MultiModalHasher.hash_kwargs( + "blake3", + model_id="test-model", + modality="image", + mm_uuid="image-uuid", + mm_content_size=width * height, + ), MultiModalHasher.hash_kwargs( "blake3", model_id="test-model", image=images[1] ), @@ -1330,4 +1337,53 @@ def test_processor_inputs_hashes_ignore_unrelated_kwargs(): hf_processor_mm_kwargs={"videos_kwargs": {"size": {"longest_edge": 448}}}, ) - assert inputs.get_mm_hashes("test-model", "blake3") == {"image": ["image-uuid"]} + width, height = image.size + assert inputs.get_mm_hashes("test-model", "blake3") == { + "image": [ + MultiModalHasher.hash_kwargs( + "blake3", + model_id="test-model", + modality="image", + mm_uuid="image-uuid", + mm_content_size=width * height, + ) + ] + } + + +def test_processor_inputs_uuid_bound_to_content_size(): + """Reusing a UUID for different-sized payloads must not share a cache key + (stale features under a fresh splice: silent wrong output when the + processed lengths match, fatal engine failure when they do not), while + the same payload under the same UUID must keep deduplicating.""" + audio_a = np.zeros(16000, dtype=np.float32) + audio_b = np.zeros(32000, dtype=np.float32) + + def hashes(audio): + return ProcessorInputs( + prompt=[], + mm_data_items=MultiModalDataParser().parse_mm_data({"audio": [audio]}), + mm_uuid_items={"audio": ["audio-uuid"]}, + ).get_mm_hashes("test-model", "blake3")["audio"][0] + + assert hashes(audio_a) == MultiModalHasher.hash_kwargs( + "blake3", + model_id="test-model", + modality="audio", + mm_uuid="audio-uuid", + mm_content_size=16000, + ) + assert hashes(audio_a) != hashes(audio_b) + assert hashes(audio_a) == hashes(audio_a.copy()) + + +def test_processor_inputs_uuid_verbatim_when_no_content_size(): + """Without a size discriminator (e.g. a cache-resident placeholder + item), the UUID is trusted as-is, preserving the existing behavior.""" + inputs = ProcessorInputs( + prompt=[], + mm_data_items={"audio": AudioProcessorItems([None])}, + mm_uuid_items={"audio": ["audio-uuid"]}, + ) + + assert inputs.get_mm_hashes("test-model", "blake3") == {"audio": ["audio-uuid"]} diff --git a/vllm/multimodal/parse.py b/vllm/multimodal/parse.py index 7a4ffc3d6df8..28155a728c5e 100644 --- a/vllm/multimodal/parse.py +++ b/vllm/multimodal/parse.py @@ -101,6 +101,18 @@ def get_item_for_hash(self, index: int) -> object: def get_all_items_for_hash(self) -> list[object]: return [self.get_item_for_hash(idx) for idx in range(self.get_count())] + def get_item_content_size(self, index: int) -> int | None: + """Cheap content discriminator for UUID-keyed cache entries. + + Binds a client-provided UUID to the item's canonical size so that + reusing a UUID for a different-sized payload cannot serve stale + cached features: that yields silent wrong output when the + processed lengths happen to match, and a fatal engine failure + when they do not (#55547). ``None`` means no discriminator is + available and the UUID is trusted as-is. + """ + return None + @abstractmethod def get_processor_data(self) -> Mapping[str, object]: """Get the data to pass to the HF processor.""" @@ -339,13 +351,20 @@ class AudioProcessorItems(ProcessorBatchItems[HfAudioItem | None]): def __init__(self, data: Sequence[HfAudioItem | None]) -> None: super().__init__(data, "audio") - def get_audio_length(self, item_idx: int) -> int: - audio = self.get(item_idx) + def get_item_content_size(self, index: int) -> int | None: + audio = self.get(index) if audio is None: - raise ValueError(f"Cannot get length of cached audio at {item_idx}") + return None return len(audio) + def get_audio_length(self, item_idx: int) -> int: + size = self.get_item_content_size(item_idx) + if size is None: + raise ValueError(f"Cannot get length of cached audio at {item_idx}") + + return size + class AudioEmbeddingItems(EmbeddingItems): def __init__( @@ -365,6 +384,13 @@ class ImageProcessorItems(ProcessorBatchItems[HfImageItem | None]): def __init__(self, data: Sequence[HfImageItem | None]) -> None: super().__init__(data, "image") + def get_item_content_size(self, index: int) -> int | None: + if self.get(index) is None: + return None + + size = self.get_image_size(index) + return size.width * size.height + def get_image_size(self, item_idx: int) -> ImageSize: image = self.get(item_idx) if image is None: @@ -418,6 +444,13 @@ def get_item_for_hash(self, index: int) -> Any: return item, metadata return item + def get_item_content_size(self, index: int) -> int | None: + if self.get(index) is None: + return None + + frame_size = self.get_frame_size(index) + return self.get_num_frames(index) * frame_size.width * frame_size.height + def get_num_frames(self, item_idx: int) -> int: video = self.get(item_idx) if video is None: diff --git a/vllm/multimodal/processing/inputs.py b/vllm/multimodal/processing/inputs.py index 56750c3087cb..ca5e2dc2c2f8 100644 --- a/vllm/multimodal/processing/inputs.py +++ b/vllm/multimodal/processing/inputs.py @@ -99,8 +99,27 @@ def get_mm_hashes( ) ) else: - # If there are no extra kwargs, use the client-provided UUID. - hashes.append(uuid_item) + # Bind the client-provided UUID to the item's content + # size so that reusing a UUID for a different payload + # cannot serve stale cached features: that yields silent + # wrong output when the processed lengths happen to + # match, and a fatal engine failure when they do not + # (#55547). Falls back to trusting the UUID as-is when + # no size discriminator is available. + content_size = data_items.get_item_content_size(i) + if content_size is not None: + hashes.append( + hasher.hash_kwargs( + hash_algorithm, + model_id=model_id, + modality=modality, + mm_uuid=uuid_item, + mm_content_size=content_size, + ) + ) + else: + # If there are no extra kwargs, use the client-provided UUID. + hashes.append(uuid_item) mm_hashes[modality] = hashes