Merge pull request #678 from DagsHub/mot_cvat_video_converter

Add MOT and CVAT video annotation import/export support

Author: deanp70

dagshub/auth/token_auth.py

@@ -37,7 +37,7 @@ def auth_flow(self, request: Request) -> Generator[Request, Response, None]:
 
     def can_renegotiate(self):
         # Env var tokens cannot renegotiate, every other token type can
-        return not type(self._token) is EnvVarDagshubToken
+        return type(self._token) is not EnvVarDagshubToken
 
     def renegotiate_token(self):
         if not self._token_storage.is_valid_token(self._token, self._host):

dagshub/data_engine/annotation/importer.py

@@ -3,6 +3,7 @@
 from dagshub_annotation_converter.formats.label_studio.task import LabelStudioTask, parse_ls_task
 from dagshub_annotation_converter.formats.yolo import YoloContext, import_lookup, import_yolo_result
 from dagshub_annotation_converter.formats.yolo.categories import Categories
+from dagshub_annotation_converter.ir.base import IRAnnotationBase, IRTaskAnnotation
 from dagshub_annotation_converter.ir.image import (
     CoordinateStyle,
     IRBBoxImageAnnotation,
@@ -11,7 +12,8 @@
     IRSegmentationImageAnnotation,
     IRSegmentationPoint,
 )
-from dagshub_annotation_converter.ir.image.annotations.base import IRAnnotationBase, IRImageAnnotationBase
+from dagshub_annotation_converter.ir.image.annotations.base import IRImageAnnotationBase
+from dagshub_annotation_converter.ir.video import IRVideoAnnotationTrack
 
 from dagshub.common.api import UserAPI
 from dagshub.common.helpers import log_message
@@ -22,6 +24,8 @@
 
     from dagshub.data_engine.model.datapoint import Datapoint
 
+from dagshub_annotation_converter.formats.label_studio.videorectangle import VideoRectangleAnnotation
+
 
 class AnnotationMetaDict(dict):
     def __init__(self, annotation: "MetadataAnnotations", *args, **kwargs):
@@ -63,13 +67,13 @@ def __init__(
         self,
         datapoint: "Datapoint",
         field: str,
-        annotations: Optional[Sequence["IRAnnotationBase"]] = None,
+        annotations: Optional[Sequence["IRTaskAnnotation"]] = None,
         meta: Optional[Dict] = None,
         original_value: Optional[bytes] = None,
     ):
         self.datapoint = datapoint
         self.field = field
-        self.annotations: list["IRAnnotationBase"]
+        self.annotations: list["IRTaskAnnotation"]
         if annotations is None:
             annotations = []
         self.annotations = list(annotations)
@@ -94,12 +98,34 @@ def to_ls_task(self) -> Optional[bytes]:
         task = LabelStudioTask(
             user_id=UserAPI.get_current_user(self.datapoint.datasource.source.repoApi.host).user_id,
         )
-        task.data["image"] = self.datapoint.download_url
-        # TODO: need to filter out non-image annotations here maybe?
-        task.add_ir_annotations(self.annotations)
+        if any(isinstance(ann, IRVideoAnnotationTrack) for ann in self.annotations):
+            task.data["video"] = self.datapoint.download_url
+            frames_count = self._get_video_frames_count()
+            for ann in self.annotations:
+                if isinstance(ann, IRVideoAnnotationTrack):
+                    ls_ann = VideoRectangleAnnotation.from_ir_track(ann, frames_count=frames_count)
+                    if ann.__pydantic_extra__ is not None:
+                        ls_ann.__pydantic_extra__ = ann.__pydantic_extra__.copy()
+                    task.add_annotation(ls_ann)
+                else:
+                    task.add_ir_annotation(ann)
+        else:
+            task.data["image"] = self.datapoint.download_url
+            task.add_ir_annotations(self.annotations)
         task.meta.update(self.meta)
         return task.model_dump_json().encode("utf-8")
 
+    def _get_video_frames_count(self) -> Optional[int]:
+        max_frame: Optional[int] = None
+        for ann in self.annotations:
+            if not isinstance(ann, IRVideoAnnotationTrack):
+                continue
+            for track_ann in ann.annotations:
+                max_frame = track_ann.frame_number if max_frame is None else max(max_frame, track_ann.frame_number)
+        if max_frame is None:
+            return None
+        return max_frame + 1
+
     @property
     def value(self) -> Optional[bytes]:
         """

dagshub/data_engine/annotation/metadata.py

@@ -0,0 +1,41 @@
+from collections import defaultdict
+from typing import Optional, Sequence
+
+from dagshub_annotation_converter.ir.video import (
+    IRVideoFrameAnnotationBase,
+    IRVideoAnnotationTrack,
+    IRVideoSequence,
+)
+
+
+def build_video_sequence_from_annotations(
+    annotations: Sequence[IRVideoFrameAnnotationBase],
+    filename: Optional[str] = None,
+) -> IRVideoSequence:
+    # Pre-group annotations into tracks (required by new from_annotations API)
+    by_track: dict[str, list[IRVideoFrameAnnotationBase]] = defaultdict(list)
+    for ann in annotations:
+        object_id = ann.imported_id
+        if object_id is None:
+            raise ValueError("Video annotation is missing an object identifier")
+        by_track[object_id].append(ann)
+
+    tracks = [
+        IRVideoAnnotationTrack.from_annotations(anns, object_id=str(tid))
+        for tid, anns in by_track.items()
+    ]
+
+    sequence = IRVideoSequence.from_annotations(tracks=tracks, filename=filename)
+
+    if filename is not None:
+        for track in sequence.tracks:
+            for ann in track.annotations:
+                if ann.filename is None:
+                    ann.filename = filename
+
+    # resolved_* methods now cache results automatically
+    sequence.resolved_video_width()
+    sequence.resolved_video_height()
+    sequence.resolved_sequence_length()
+
+    return sequence

dagshub/data_engine/annotation/video.py

@@ -1668,6 +1668,16 @@ def import_annotations_from_files(
 
         Keyword Args:
             yolo_type: Type of YOLO annotations to import. Either ``bbox``, ``segmentation`` or ``pose``.
+            image_width: (MOT, CVAT video) Width of the video frames in pixels. \
+                Used when the annotation file does not contain dimension metadata.
+            image_height: (MOT, CVAT video) Height of the video frames in pixels. \
+                Used when the annotation file does not contain dimension metadata.
+            video_name: (MOT) Name/path of the video file these annotations belong to. \
+                Used to key the resulting annotations when it cannot be inferred from the annotation file.
+            video_dir_name: (MOT filesystem layout) Name of the subdirectory containing video files. \
+                Defaults to ``"videos"``.
+            label_dir_name: (MOT filesystem layout) Name of the subdirectory containing label files. \
+                Defaults to ``"labels"``.
 
         Example to import segmentation annotations into an ``imported_annotations`` field,
         using YOLO information from an ``annotations.yaml`` file (can be local, or in the repo)::