Fix type error when the visual encoder is not CLIP (#496)

* Fix type error when the visual encoder is not CLIP.

* update

Author: hhaAndroid

xtuner/dataset/llava.py

@@ -98,7 +98,10 @@ def __getitem__(self, index):
                 image, return_tensors='pt')['pixel_values'][0]
             data_dict['pixel_values'] = image
         else:
-            crop_size = self.image_processor.crop_size
+            if hasattr(self.image_processor, 'crop_size'):
+                crop_size = self.image_processor.crop_size
+            else:
+                crop_size = self.image_processor.size
             data_dict['pixel_values'] = torch.zeros(3, crop_size['height'],
                                                     crop_size['width'])
         return data_dict

xtuner/engine/hooks/evaluate_chat_hook.py

@@ -129,7 +129,8 @@ def _eval_images(self,
                     input_ids.append(IMAGE_TOKEN_INDEX)
             input_ids = torch.tensor(input_ids).to(device)
             visual_outputs = model.visual_encoder(
-                image.unsqueeze(0), output_hidden_states=True)
+                image.unsqueeze(0).to(model.visual_encoder.dtype),
+                output_hidden_states=True)
             pixel_values = model.projector(
                 visual_outputs.hidden_states[model.visual_select_layer][:, 1:])
 

xtuner/model/llava.py

@@ -169,7 +169,8 @@ def _build_from_cfg_or_module(self, cfg_or_mod):
     def forward(self, data, data_samples=None, mode='loss'):
         if 'pixel_values' in data:
             visual_outputs = self.visual_encoder(
-                data['pixel_values'], output_hidden_states=True)
+                data['pixel_values'].to(self.visual_encoder.dtype),
+                output_hidden_states=True)
             pixel_values = self.projector(
                 visual_outputs.hidden_states[self.visual_select_layer][:, 1:])
             data['pixel_values'] = pixel_values

xtuner/tools/chat.py

@@ -306,7 +306,7 @@ def main():
                 image, tuple(int(x * 255) for x in image_processor.image_mean))
             image = image_processor.preprocess(
                 image, return_tensors='pt')['pixel_values'][0]
-            image = image.cuda().unsqueeze(0)
+            image = image.cuda().unsqueeze(0).to(visual_encoder.dtype)
             visual_outputs = visual_encoder(image, output_hidden_states=True)
             pixel_values = projector(
                 visual_outputs.hidden_states[args.visual_select_layer][:, 1:])

xtuner/tools/mmbench.py

@@ -445,7 +445,7 @@ def main():
             image, tuple(int(x * 255) for x in image_processor.image_mean))
         image = image_processor.preprocess(
             image, return_tensors='pt')['pixel_values'][0]
-        image = image.cuda().unsqueeze(0)
+        image = image.cuda().unsqueeze(0).to(visual_encoder.dtype)
         visual_outputs = visual_encoder(image, output_hidden_states=True)
         pixel_values = projector(
             visual_outputs.hidden_states[args.visual_select_layer][:, 1:])