fix video mm (#105)

Author: tastelikefeet

src/twinkle/processor/base.py

@@ -311,9 +311,13 @@ def to_transformers_dict(inputs: List[InputFeature], **kwargs) -> List[InputFeat
         for _input in inputs:
             output = {}
             _keys = [
-                'input_ids', 'input_embeddings', 'attention_mask', 'position_ids', 'labels', 'completion_mask',
-                'pixel_values', 'image_grid_thw'
-            ]
+                'input_ids',
+                'input_embeddings',
+                'attention_mask',
+                'position_ids',
+                'labels',
+                'completion_mask',
+            ] + list(InputProcessor.VLM_CONCAT_FIELDS)
             for key in list(_input.keys()):
                 if key in _keys:
                     output[key] = np.array(_input[key]) if not isinstance(_input[key], torch.Tensor) else _input[key]

src/twinkle/template/base.py

@@ -226,7 +226,7 @@ def _build_mm_messages(self, trajectory: Trajectory) -> List[Trajectory]:
                 message['images'] = self.preprocess_images(msg_images)
                 assert len(message['images']) == content.count(self.image_placeholder)
             if msg_videos:
-                message['videos'] = self.preprocess_images(msg_videos)
+                message['videos'] = self.preprocess_videos(msg_videos)
                 assert len(message['videos']) == content.count(self.video_placeholder)
             if msg_audios:
                 message['audios'] = self.preprocess_audios(msg_audios)