[data] fix: avoid duplicate BOS after chat template rendering

loongforge/data/multimodal/llava_ov_task_encoder.py

@@ -183,10 +183,10 @@ def encode_vqa(self, sample: VQASample) -> BaseTaskSample:
             if text[-1] == "\n":
                 text = text[:-1]
             input_ids, _, imgs, image_grid_thw, attn_mask = self._process(
-                sample.image, text
+                sample.image, text, add_special_tokens=False
             )
             target = torch.ones_like(input_ids) * IGNORE_INDEX
-            answers = self.tokenizer.tokenize(sample.answers)
+            answers = self.tokenizer.tokenize(sample.answers, add_special_tokens=False)
             target[-len(answers) - 1 : -1] = torch.tensor(answers)
             target[-1] = input_ids[-1]
             # print(target[-1])

loongforge/data/multimodal/vlm_task_encoder.py

@@ -222,13 +222,14 @@ def _resize_image(self, image, size_factor=28):
 
         return image
 
-    def _process(self, image, text):
+    def _process(self, image, text, add_special_tokens=True):
         """ " Process the data to get the model's input"""
         inputs = self.processor(
             text=text,
             images=image,
             padding=True,
             return_tensors="pt",
+            add_special_tokens=add_special_tokens,
         )
         input_ids = inputs["input_ids"][0]
         attn_mask = inputs["attention_mask"][0].logical_not()
@@ -261,9 +262,11 @@ def process_sft_vqa(self, context, answer, image):
         ).replace("<image>", IMAGE_TOKEN_WITH_TAGS)
         if text[-1] == "\n":
             text = text[:-1]
-        input_ids, _, imgs, image_grid_thw, attn_mask = self._process(image, text)
+        input_ids, _, imgs, image_grid_thw, attn_mask = self._process(
+            image, text, add_special_tokens=False
+        )
         target = torch.ones_like(input_ids) * IGNORE_INDEX
-        answer_ids = self.tokenizer.tokenize(answer)
+        answer_ids = self.tokenizer.tokenize(answer, add_special_tokens=False)
         target[-len(answer_ids) - 1 : -1] = torch.tensor(answer_ids)
 
         return input_ids, target, attn_mask, imgs, image_grid_thw
@@ -908,10 +911,10 @@ def encode_vqa4packing(self, sample: VQASample) -> BaseTaskSample:
             pass
 
         input_ids, _, imgs, image_grid_thw, attn_mask = self._process(
-            sample.image, text
+            sample.image, text, add_special_tokens=False
         )
         target = torch.ones_like(input_ids) * IGNORE_INDEX
-        answers = self.tokenizer.tokenize(sample.answers)
+        answers = self.tokenizer.tokenize(sample.answers, add_special_tokens=False)
         target[-len(answers) - 1 : -1] = torch.tensor(answers)
         target[-1] = input_ids[-1]
 

loongforge/embodied/data/datasets/groot_n1_6/transforms/processor_groot_n1_6.py

@@ -561,6 +561,7 @@ def __call__(self, features: list[dict[str, Any]]) -> BatchFeature:
                     padding="max_length" if self.max_length else True,
                     max_length=self.max_length,
                     truncation=self.max_length is not None,
+                    add_special_tokens=False,
                 )
                 # Detect truncation: if any sample has attention_mask all-1 with
                 # max_length set, it was truncated (no pad tokens added).

loongforge/embodied/data/datasets/groot_n1_7/transforms/groot_collator.py

@@ -105,6 +105,7 @@ def __call__(self, features: list[dict[str, Any]]) -> BatchFeature:
                     padding="max_length" if self.max_length else True,
                     max_length=self.max_length,
                     truncation=self.max_length is not None,
+                    add_special_tokens=False,
                 )
                 for vlm_key, vlm_value in vlm_inputs.items():
                     batch[vlm_key] = vlm_value