Add exec and invoke duration settings

Author: jmsmkn

app/grandchallenge/components/backends/base.py

@@ -325,11 +325,15 @@ def __init__(
             use_warm_pool and settings.COMPONENTS_USE_WARM_POOL
         )
         self._signing_key = signing_key
+        self._algorithm_model = algorithm_model
+        self._ground_truth = ground_truth
+
+        self._exec_duration = None
+        self._invoke_duration = None
         self._stdout = []
         self._stderr = []
+
         self.__s3_client = None
-        self._algorithm_model = algorithm_model
-        self._ground_truth = ground_truth
 
     def provision(self, *, input_civs, input_prefixes):
         # We cannot run everything async as it requires database access.
@@ -401,6 +405,14 @@ def stderr(self):
     @abstractmethod
     def utilization_duration(self): ...
 
+    @property
+    def exec_duration(self):
+        return self._exec_duration
+
+    @property
+    def invoke_duration(self):
+        return self._invoke_duration
+
     @property
     @abstractmethod
     def usd_cents_per_hour(self): ...
@@ -866,6 +878,9 @@ def _get_inference_result(self):
     def _handle_completed_job(self):
         inference_result = self._get_inference_result()
 
+        self._exec_duration = inference_result.exec_duration
+        self._invoke_duration = inference_result.invoke_duration
+
         users_process_exit_code = inference_result.return_code
 
         if users_process_exit_code == 0:

app/grandchallenge/components/models.py

@@ -1736,7 +1736,7 @@ def save(self, *args, **kwargs):
         if adding:
             self.create_utilization()
 
-    def update_status(
+    def update_status(  # noqa:C901
         self,
         *,
         status: STATUS_CHOICES,
@@ -1745,6 +1745,8 @@ def update_status(
         error_message="",
         detailed_error_message=None,
         utilization_duration=None,
+        exec_duration=None,
+        invoke_duration=None,
         compute_cost_euro_millicents=None,
         runtime_metrics=None,
     ):
@@ -1769,6 +1771,12 @@ def update_status(
             self.utilization.duration = utilization_duration
             self.utilization.save(update_fields=["duration"])
 
+        if exec_duration is not None:
+            self.exec_duration = exec_duration
+
+        if invoke_duration is not None:
+            self.invoke_duration = invoke_duration
+
         if compute_cost_euro_millicents is not None:
             self.utilization.compute_cost_euro_millicents = (
                 compute_cost_euro_millicents

app/grandchallenge/components/tasks.py

@@ -858,6 +858,8 @@ def get_update_status_kwargs(*, executor=None):
             "stdout": executor.stdout,
             "stderr": executor.stderr,
             "utilization_duration": executor.utilization_duration,
+            "exec_duration": executor.exec_duration,
+            "invoke_duration": executor.invoke_duration,
             "compute_cost_euro_millicents": executor.compute_cost_euro_millicents,
             "runtime_metrics": executor.runtime_metrics,
         }

app/tests/algorithms_tests/test_tasks.py

@@ -338,25 +338,26 @@ def test_algorithm(client, settings, django_capture_on_commit_callbacks):
         django_capture_on_commit_callbacks=django_capture_on_commit_callbacks,
     )
 
-    jobs = Job.objects.filter(algorithm_image=ai).all()
-
     # There should be a single, successful job
-    assert len(jobs) == 1
+    job = Job.objects.filter(algorithm_image=ai).get()
 
-    assert jobs[0].stdout.endswith("Greetings from stdout")
-    assert jobs[0].stderr.endswith('("Hello from stderr")')
-    assert "UserWarning: Could not google: [Errno " in jobs[0].stderr
-    assert jobs[0].error_message == ""
-    assert jobs[0].status == jobs[0].SUCCESS
+    assert job.stdout.endswith("Greetings from stdout")
+    assert job.stderr.endswith('("Hello from stderr")')
+    assert "UserWarning: Could not google: [Errno " in job.stderr
+    assert job.error_message == ""
+    assert job.status == job.SUCCESS
+    assert job.exec_duration == timedelta(seconds=1337)
+    assert job.invoke_duration == timedelta(seconds=1874)
+    assert job.utilization.duration.total_seconds() > 0
 
     # The job should have two ComponentInterfaceValues,
     # one for the results.json and one for output.tif
-    assert len(jobs[0].outputs.all()) == 2
+    assert len(job.outputs.all()) == 2
 
-    json_result_civ = jobs[0].outputs.get(interface=json_result_interface)
+    json_result_civ = job.outputs.get(interface=json_result_interface)
     assert json_result_civ.value
 
-    heatmap_civ = jobs[0].outputs.get(interface=heatmap_interface)
+    heatmap_civ = job.outputs.get(interface=heatmap_interface)
     assert heatmap_civ.image.name == "input_file.tif"
 
     # We add another ComponentInterface with file value and run the algorithm again

app/tests/components_tests/resources/backends.py

@@ -90,7 +90,7 @@ def execute(self):
                 pk=self._job_id,
                 return_code=0,
                 exec_duration=timedelta(seconds=1337),
-                invoke_duration=None,
+                invoke_duration=timedelta(seconds=1874),
                 outputs=[],
                 sagemaker_shim_version="0.5.0",
             )
@@ -124,6 +124,8 @@ def execute(self):
                         "_job_id": self._job_id,
                         "_stdout": self._stdout,
                         "_stderr": self._stderr,
+                        "_exec_duration_seconds": self._exec_duration.total_seconds(),
+                        "_invoke_duration_seconds": self._invoke_duration.total_seconds(),
                         "__start_time": self.__start_time,
                     },
                     "backend": f"{self.__class__.__module__}.{self.__class__.__qualname__}",
@@ -134,6 +136,12 @@ def execute(self):
     def handle_event(self, *, event):
         self._stdout = event["_stdout"]
         self._stderr = event["_stderr"]
+        self._exec_duration = timedelta(
+            seconds=event["_exec_duration_seconds"]
+        )
+        self._invoke_duration = timedelta(
+            seconds=event["_invoke_duration_seconds"]
+        )
         self.__start_time = event["__start_time"]
 
     @staticmethod

app/tests/components_tests/test_amazon_sagemaker_training_backend.py

@@ -563,8 +563,8 @@ def test_handle_completed_job():
     inference_result = InferenceResult(
         pk=f"algorithms-job-{pk}",
         return_code=0,
-        exec_duration=timedelta(seconds=1337),
-        invoke_duration=None,
+        exec_duration=timedelta(seconds=51432),
+        invoke_duration=timedelta(seconds=1543),
         outputs=[],
         sagemaker_shim_version="0.5.0",
     )
@@ -587,6 +587,10 @@ def test_handle_completed_job():
 
     assert executor._handle_completed_job() is None
 
+    # The durations should be set from the result object
+    assert executor.exec_duration == timedelta(seconds=51432)
+    assert executor.invoke_duration == timedelta(seconds=1543)
+
 
 def test_handle_time_limit_exceded(settings):
     settings.COMPONENTS_AMAZON_ECR_REGION = "us-east-1"

app/tests/evaluation_tests/test_tasks.py

@@ -130,6 +130,9 @@ def test_submission_evaluation(
         ]
         == 0.5
     )
+    assert evaluation.exec_duration == timedelta(seconds=1337)
+    assert evaluation.invoke_duration == timedelta(seconds=1874)
+    assert evaluation.utilization.duration.total_seconds() > 0
 
     # Try with a csv file
     user_upload = create_upload_from_file(