fix: Add duration from encodejob to video (#1212)

* added migration to add duration in video from encode jobs

* fixed attribute error

* fixed formatting issues

* added condition to migrate video duration data

* fixed failing test

* meeting linter

* fixed failing test for refresh status

* added backpopulation of duration in the new encoded videos

* fixed attribute errors

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* fixed video id for backpopulation

* reverted removing update_status

* added exception handling for literal evaluation and added null check for messages in refactor

* isolated back-population of new encode jobs

* fixed boundary error

* fixed more unbound variables errors

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: umar8hassan

cloudsync/api.py

@@ -19,6 +19,7 @@
 from mitol.transcoding.api import media_convert_job
 
 from odl_video import logging
+from ui.api import get_duration_from_encode_job
 from ui.constants import VideoStatus
 from ui.encodings import EncodingNames
 from ui.models import (
@@ -78,6 +79,7 @@ def process_transcode_results(results: dict) -> None:
         elif "FILE_GROUP" in group_type:
             process_mp4_outputs(group.get("outputDetails", []), video)
 
+    video.duration = get_duration_from_encode_job(results)
     video.update_status(VideoStatus.COMPLETE)
 
     # Ensure content_type and object_id are set for the EncodeJob

cloudsync/api_test.py

@@ -139,7 +139,7 @@ def test_refresh_status_video_job_othererror(mocker, status):
     mocker.patch("cloudsync.api.boto3", MockBoto)
     error = Exception("unexpected exception")
     mocker.patch(
-        "cloudsync.api.media_convert_job", return_value=MockClientMC(error=error)
+        "cloudsync.api.get_media_convert_job", return_value=MockClientMC(error=error)
     )
     with pytest.raises(Exception):
         api.refresh_status(video)

ui/api.py

@@ -93,12 +93,7 @@ def post_video_to_edx(video_files):
     for edx_endpoint in edx_endpoints:
         try:
             edx_endpoint.refresh_access_token()
-            encode_job = (
-                video_files[0]
-                .video.encode_jobs.filter(state=models.EncodeJob.State.COMPLETED)
-                .first()
-            )
-            duration = get_duration_from_encode_job(encode_job)
+            duration = video_files[0].video.duration
             video_key = str(video_files[0].video.key)
             resp = requests.post(
                 edx_endpoint.full_api_url,
@@ -163,11 +158,7 @@ def update_video_on_edx(video_key, encoded_videos=None):
             payload = {
                 "edx_video_id": str(video.key),
                 "client_video_id": video.title,
-                "duration": get_duration_from_encode_job(
-                    video.encode_jobs.filter(
-                        state=models.EncodeJob.State.COMPLETED
-                    ).first()
-                ),
+                "duration": video.duration,
                 "status": "updated",
             }
             if encoded_videos:
@@ -197,8 +188,8 @@ def get_duration_from_encode_job(encode_job):
         duration: float
     """
     duration = 0.0
-    if encode_job and encode_job.message:
-        if output_groups := encode_job.message.get("outputGroupDetails", []):
+    if encode_job:
+        if output_groups := encode_job.get("outputGroupDetails", []):
             # Get the first output group
             output_group = output_groups[0]
             if outputs := output_group.get("outputDetails", []):

ui/api_test.py

@@ -18,8 +18,6 @@
 from ui.factories import (
     CollectionEdxEndpointFactory,
     CollectionFactory,
-    EncodeJobFactory,
-    VideoFactory,
     VideoFileFactory,
 )
 
@@ -378,9 +376,7 @@ def test_get_duration_from_encode_job():
     """
     get_duration_from_encode_job should return duration from video's encode_jobs message body
     """
-    video = VideoFactory(status="Complete")
-    encode_job = EncodeJobFactory(video=video)
-    encode_job.message = {
+    encode_job = {
         "id": "1711563064503-e5qdnh",
         "outputGroupDetails": [
             {

ui/migrations/0036_video_duration.py

@@ -0,0 +1,87 @@
+# Generated by Django 4.2.21 on 2025-05-23 01:10
+
+from ast import literal_eval
+from django.db import migrations, models, connection
+
+
+def migrate_video_duration(apps, schema_editor):
+    """
+    Migrate video duration from the old encode job if data exists.
+    """
+
+    db_alias = schema_editor.connection.alias
+    EncodeJob = apps.get_model("ui", "EncodeJob")
+    Video = apps.get_model("ui", "Video")
+
+    if (
+        "dj_elastictranscoder_encodejob"
+        in schema_editor.connection.introspection.table_names()
+    ):
+        with connection.cursor() as cursor:
+            cursor.execute(
+                """
+                SELECT e1.object_id, e1.message
+                FROM dj_elastictranscoder_encodejob e1
+                INNER JOIN (
+                    SELECT object_id, MAX(created_at) as max_created_at
+                    FROM dj_elastictranscoder_encodejob
+                    WHERE state = 4
+                    GROUP BY object_id
+                ) e2 ON e1.object_id = e2.object_id AND e1.created_at = e2.max_created_at
+                WHERE e1.state = 4
+                """
+            )
+            rows = cursor.fetchall()
+
+        for row in rows:
+            video_id, message = row
+            try:
+                # Attempt to parse the message as a literal
+                message = literal_eval(message)
+            except (ValueError, SyntaxError, TypeError):
+                # If parsing fails, skip this row
+                continue
+
+            if duration := (
+                message
+                and isinstance(message, dict)
+                and message.get("Output", {}).get("Duration")
+            ):
+                Video.objects.using(db_alias).filter(id=video_id).update(
+                    duration=duration
+                )
+
+    encode_jobs = EncodeJob.objects.using(db_alias).filter(state=4)
+    for job in encode_jobs:
+        video_id = job.object_id
+        if not video_id:
+            continue
+        # Fetch the video object
+        video = Video.objects.using(db_alias).get(id=video_id)
+        if video and (not video.duration):
+            if output_groups := job.message.get("outputGroupDetails", []):
+                # Get the first output group
+                output_group = output_groups[0]
+                if outputs := output_group.get("outputDetails", []):
+                    # Get the first output
+                    output = outputs[0]
+                    duration_in_ms = output.get("durationInMs", 0)
+                    # Convert milliseconds to seconds
+                    duration = duration_in_ms / 1000.0
+                    video.duration = duration
+                    video.save(update_fields=["duration"])
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("ui", "0035_encodejob"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="video",
+            name="duration",
+            field=models.FloatField(default=0.0, null=True),
+        ),
+        migrations.RunPython(migrate_video_duration, migrations.RunPython.noop),
+    ]

ui/models.py

@@ -351,6 +351,7 @@ class Video(TimestampedModel):
     is_logged_in_only = models.BooleanField(null=False, default=False)
     custom_order = models.IntegerField(null=True, blank=True)
     schedule_retranscode = models.BooleanField(default=False)
+    duration = models.FloatField(null=True, default=0.0)
 
     objects = VideoManager()