other: Update the version of transformers to 4.53.1 (#102)

Author: rebel-eunji

requirements.txt

@@ -3,6 +3,5 @@ setuptools-scm>=8
 pyyaml
 datasets
 qwen_vl_utils
-transformers==4.51.3
 vllm==0.9.1
-optimum-rbln>=0.9.1
+optimum-rbln>=0.9.2a2

vllm_rbln/model_executor/models/optimum/llava.py

@@ -54,20 +54,18 @@ def _forward(
         input_ids: torch.LongTensor = None,
         pixel_values: torch.FloatTensor = None,
         image_sizes: Optional[torch.LongTensor] = None,
-        inputs_embeds: Optional[torch.FloatTensor] = None,
-        vision_feature_layer: Optional[int] = None,
-        vision_feature_select_strategy: Optional[str] = None,
         cache_position: Union[List[torch.Tensor],
                               torch.Tensor] = None,  # vllm keyword argument
         **kwargs,
     ):
         if is_prefill:
+            # NOTE inputs_embeds will be generated inside _preprocess_prefill
             inputs_embeds = self.model._preprocess_prefill(
                 input_ids=input_ids,
-                inputs_embeds=inputs_embeds,
                 pixel_values=pixel_values,
                 image_sizes=image_sizes,
             )
+
             if self.model.language_model.prefill_decoder is None:
                 raise version_error
 
@@ -131,9 +129,9 @@ def forward(self, model_input: ModelInputForRBLN,
             is_prefill=is_prompt,
             block_tables=block_tables,
             input_ids=input_ids,
-            cache_position=cache_position,
             pixel_values=pixel_values,
             image_sizes=image_sizes,
+            cache_position=cache_position,
         )
 
         if not is_prompt:

vllm_rbln/model_executor/models/optimum/llava_next.py

@@ -71,35 +71,18 @@ def _forward(
         input_ids: torch.LongTensor = None,
         pixel_values: torch.FloatTensor = None,
         image_sizes: Optional[torch.LongTensor] = None,
-        inputs_embeds: Optional[torch.FloatTensor] = None,
-        vision_feature_layer: Optional[int] = None,
-        vision_feature_select_strategy: Optional[str] = None,
         cache_position: Union[List[torch.Tensor],
                               torch.Tensor] = None,  # vllm keyword argument
         **kwargs,
     ):
-        if inputs_embeds is not None:
-            raise NotImplementedError(
-                "Specifying inputs_embeds is not supported.")
-
         if is_prefill:
-            # Get text_embeds
-            inputs_embeds = self.model.text_embedding(input_ids)
-
-            # If any images in the prompt, get image_embeds and merge with text
-            if pixel_values is not None and input_ids.shape[
-                    1] != 1 and pixel_values.size(0) > 0:
-                image_features, _ = self.model.image_embedding(
-                    image_sizes, pixel_values, vision_feature_layer,
-                    vision_feature_select_strategy)
-
-                inputs_embeds = self.merge_multimodal_embeddings(
-                    input_ids, inputs_embeds, image_features,
-                    self.model.config.image_token_index)
-        else:
-            inputs_embeds = self.model.text_embedding(input_ids=input_ids)
+            # NOTE inputs_embeds will be generated inside _preprocess_prefill
+            inputs_embeds = self.model._preprocess_prefill(
+                input_ids=input_ids,
+                pixel_values=pixel_values,
+                image_sizes=image_sizes,
+            )
 
-        if is_prefill:
             if self.model.language_model.prefill_decoder is None:
                 raise version_error
 
@@ -113,7 +96,7 @@ def _forward(
                 raise version_error
 
             logits = self.model.language_model.decoder(
-                inputs_embeds=inputs_embeds,
+                input_ids=input_ids,
                 cache_position=cache_position,
                 block_tables=block_tables,
             ).logits
@@ -160,9 +143,9 @@ def forward(self, model_input: ModelInputForRBLN,
             is_prefill=is_prompt,
             block_tables=block_tables,
             input_ids=input_ids,
-            cache_position=cache_position,
             pixel_values=pixel_values,
             image_sizes=image_sizes,
+            cache_position=cache_position,
         )
 
         if not is_prompt: