Link ComponentJobUtilization and Invoice

app/grandchallenge/algorithms/tasks.py

@@ -10,6 +10,7 @@
 from django.utils import timezone
 
 from grandchallenge.algorithms.exceptions import TooManyJobsScheduled
+from grandchallenge.challenges.exceptions import InsufficientBudgetError
 from grandchallenge.components.schemas import GPUTypeChoices
 from grandchallenge.components.tasks import (
     provision_invocation_input_data,
@@ -165,6 +166,21 @@ def create_algorithm_jobs(
                 input_civ_set=ai.values.all(),
                 use_warm_pool=use_warm_pool,
             )
+            jobs.append(
+                job
+            )  # Keep track of created jobs, even if utilization setup has failed
+
+            if job_utilization_challenge is not None:
+                try:
+                    job.utilization.invoice = (
+                        job_utilization_challenge.active_invoice
+                    )
+                except InsufficientBudgetError:
+                    job.update_status(
+                        status=Job.FAILURE,
+                        error_message="Job cannot be executed. The challenge has insufficient budget to run this job.",
+                    )
+                    break
 
             job.utilization.archive = ai.archive
             job.utilization.phase = job_utilization_phase
@@ -173,8 +189,6 @@ def create_algorithm_jobs(
 
             job.execute()
 
-            jobs.append(job)
-
     return jobs
 
 

app/grandchallenge/challenges/costs.py

@@ -20,19 +20,20 @@
 )
 
 
-def annotate_compute_costs(*, challenge):
+def annotate_compute_costs(*, invoice):
     algorithm_job_utilizations = JobUtilization.objects.filter(
-        challenge=challenge
+        invoice=invoice,
     )
     job_warm_pool_utilizations = JobWarmPoolUtilization.objects.filter(
-        challenge=challenge
+        invoice=invoice,
     )
     evaluation_job_utilizations = EvaluationUtilization.objects.filter(
-        challenge=challenge
+        invoice=invoice,
     )
 
     update_compute_cost_euro_millicents(
-        obj=challenge,
+        obj=invoice,
+        obj_field="compute_costs_utilized_euros_millicents",
         algorithm_job_utilizations=algorithm_job_utilizations,
         job_warm_pool_utilizations=job_warm_pool_utilizations,
         evaluation_job_utilizations=evaluation_job_utilizations,
@@ -54,6 +55,7 @@ def annotate_job_duration_and_compute_costs(*, phase):
 
     update_compute_cost_euro_millicents(
         obj=phase,
+        obj_field="compute_cost_euro_millicents",
         algorithm_job_utilizations=algorithm_job_utilizations,
         job_warm_pool_utilizations=job_warm_pool_utilizations,
         evaluation_job_utilizations=evaluation_job_utilizations,
@@ -63,6 +65,7 @@ def annotate_job_duration_and_compute_costs(*, phase):
 def update_compute_cost_euro_millicents(
     *,
     obj,
+    obj_field,
     algorithm_job_utilizations,
     job_warm_pool_utilizations,
     evaluation_job_utilizations,
@@ -79,8 +82,10 @@ def update_compute_cost_euro_millicents(
 
     items = [algorithm_job_costs, job_warm_pool_costs, evaluation_costs]
 
-    obj.compute_cost_euro_millicents = sum(
-        item["compute_cost_euro_millicents__sum"] or 0 for item in items
+    setattr(
+        obj,
+        obj_field,
+        sum(item["compute_cost_euro_millicents__sum"] or 0 for item in items),
     )
 
 

app/grandchallenge/challenges/exceptions.py

@@ -0,0 +1,2 @@
+class InsufficientBudgetError(Exception):
+    pass

app/grandchallenge/challenges/models.py

@@ -43,6 +43,7 @@
     send_challenge_requested_email_to_reviewers,
     send_email_percent_budget_consumed_alert,
 )
+from grandchallenge.challenges.exceptions import InsufficientBudgetError
 from grandchallenge.challenges.utils import ChallengeTypeChoices
 from grandchallenge.components.models import APIMethodChoices
 from grandchallenge.components.schemas import GPUTypeChoices
@@ -937,6 +938,22 @@ def has_paid_prepaid_invoice(self):
             payment_status=PaymentStatusChoices.PAID,
         ).exists()
 
+    @property
+    def active_invoice(self):
+        invoices = self.invoices.order_by("expires_on", "created")
+
+        total_balance = sum(
+            invoice.compute_costs_balance_euros_millicents
+            for invoice in invoices
+        )
+
+        if total_balance <= 0:
+            raise InsufficientBudgetError
+
+        for invoice in invoices:
+            if invoice.compute_costs_balance_euros_millicents > 0:
+                return invoice
+
 
 class ChallengeUserObjectPermission(UserObjectPermissionBase):
     allowed_permissions = frozenset()

app/grandchallenge/challenges/tasks.py

@@ -6,7 +6,7 @@
 from django.conf import settings
 from django.contrib.auth import get_user_model
 from django.db import transaction
-from django.db.models import Count, Max, Min, Q
+from django.db.models import Count, Max, Min, Q, Sum
 from django.utils.timezone import datetime, now
 from psycopg.errors import LockNotAvailable
 
@@ -31,6 +31,7 @@
     acks_late_micro_short_task,
 )
 from grandchallenge.evaluation.models import Evaluation, Phase
+from grandchallenge.invoices.models import Invoice
 
 
 @acks_late_2xlarge_task
@@ -102,17 +103,28 @@ def wrapper(*args, **kwargs):
 
 @acks_late_2xlarge_task
 def update_challenge_compute_costs():
-    for challenge in Challenge.objects.with_available_compute().iterator(
-        chunk_size=1000
-    ):
+    for invoice in Invoice.objects.iterator(chunk_size=1000):
         with transaction.atomic():
-            annotate_compute_costs(challenge=challenge)
+            annotate_compute_costs(invoice=invoice)
 
             @retry_with_backoff((LockNotAvailable,))
-            def save_challenge():
+            def save_invoice():
+                invoice.save(
+                    update_fields=("compute_costs_utilized_euros_millicents",)
+                )
+
+                # TODO: remove this once compute_cost_euro_millicents is removed or deprecated
+                challenge = Challenge.objects.with_available_compute().get(
+                    pk=invoice.challenge_id
+                )
+                challenge.compute_cost_euro_millicents = (
+                    Invoice.objects.filter(challenge=challenge.pk).aggregate(
+                        total=Sum("compute_costs_utilized_euros_millicents")
+                    )["total"]
+                )
                 challenge.save(update_fields=("compute_cost_euro_millicents",))
 
-            save_challenge()
+            save_invoice()
 
     for phase in Phase.objects.iterator(chunk_size=1000):
         with transaction.atomic():

app/grandchallenge/evaluation/admin.py

@@ -3,6 +3,7 @@
 from django.forms import ModelForm
 from django.utils.html import format_html
 
+from grandchallenge.challenges.exceptions import InsufficientBudgetError
 from grandchallenge.components.admin import (
     ComponentImageAdmin,
     cancel_jobs,
@@ -169,7 +170,19 @@ def reevaluate_submissions(modeladmin, request, queryset):
                 messages.WARNING,
             )
         else:
-            submission.create_evaluation(additional_inputs=None)
+            try:
+                invoice = submission.phase.challenge.active_invoice
+            except InsufficientBudgetError:
+                modeladmin.message_user(
+                    request,
+                    f"Submission {submission.pk} cannot be reevaluated. Challenge has insufficient budget.",
+                    messages.WARNING,
+                )
+            else:
+                submission.create_evaluation(
+                    additional_inputs=None,
+                    invoice=invoice,
+                )
 
 
 @admin.register(Submission)

app/grandchallenge/evaluation/forms.py

@@ -24,7 +24,7 @@
 
 from grandchallenge.algorithms.forms import UserAlgorithmsForPhaseMixin
 from grandchallenge.algorithms.models import Job
-from grandchallenge.challenges.models import Challenge
+from grandchallenge.challenges.exceptions import InsufficientBudgetError
 from grandchallenge.components.forms import (
     AdditionalInputsMixin,
     ContainerImageForm,
@@ -471,6 +471,15 @@ def clean(self):
                 "You must confirm that you want to submit to this phase."
             )
 
+        try:
+            invoice = self._phase.challenge.active_invoice
+        except InsufficientBudgetError:
+            raise ValidationError(
+                "Challenge has insufficient budget. Please contact the challenge organizers."
+            )
+        else:
+            cleaned_data["invoice"] = invoice
+
         return cleaned_data
 
     def clean_phase(self):
@@ -655,7 +664,8 @@ def save(self, *args, **kwargs):
         instance = super().save(*args, **kwargs)
 
         instance.create_evaluation(
-            additional_inputs=self.cleaned_data["additional_inputs"]
+            additional_inputs=self.cleaned_data["additional_inputs"],
+            invoice=self.cleaned_data["invoice"],
         )
 
         return instance
@@ -741,17 +751,14 @@ def clean(self):
                     "Please wait for the other evaluation to complete."
                 )
 
-        # Fetch from the db to get the cost annotations
-        challenge = (
-            Challenge.objects.filter(
-                pk=cleaned_data["submission"].phase.challenge.pk
+        try:
+            invoice = cleaned_data["submission"].phase.challenge.active_invoice
+        except InsufficientBudgetError:
+            raise ValidationError(
+                "Challenge has insufficient budget. Please contact support to add more funds."
             )
-            .with_available_compute()
-            .get()
-        )
-
-        if challenge.available_compute_euro_millicents <= 0:
-            raise ValidationError("This challenge has exceeded its budget")
+        else:
+            cleaned_data["invoice"] = invoice
 
         if Evaluation.objects.get_evaluations_with_same_inputs(
             inputs=cleaned_data["additional_inputs"],

app/grandchallenge/evaluation/models.py

@@ -1651,7 +1651,7 @@ def save(self, *args, **kwargs):
                     send_action=False,
                 )
 
-    def create_evaluation(self, *, additional_inputs):
+    def create_evaluation(self, *, additional_inputs, invoice):
         if (
             self.phase.additional_evaluation_inputs.exists()
             and not additional_inputs
@@ -1712,6 +1712,8 @@ def create_evaluation(self, *, additional_inputs):
                 requires_memory_gb=self.phase.evaluation_requires_memory_gb,
                 status=Evaluation.VALIDATING_INPUTS,
             )
+            evaluation.utilization.invoice = invoice
+            evaluation.utilization.save()
 
         if self.phase.submission_kind == SubmissionKindChoices.ALGORITHM:
             if not self.has_matching_algorithm_interfaces:

app/grandchallenge/evaluation/views/__init__.py

@@ -535,7 +535,8 @@ def get_success_url(self):
     def form_valid(self, form):
         redirect = super().form_valid(form)
         self.submission.create_evaluation(
-            additional_inputs=form.cleaned_data["additional_inputs"]
+            additional_inputs=form.cleaned_data["additional_inputs"],
+            invoice=form.cleaned_data["invoice"],
         )
         return redirect
 

app/grandchallenge/utilization/admin.py

@@ -87,6 +87,7 @@ class JobUtilizationAdmin(admin.ModelAdmin):
         "creator",
         "duration",
         "compute_cost_euro_millicents",
+        "invoice",
         "phase",
         "challenge",
         "algorithm_image",
@@ -95,6 +96,7 @@ class JobUtilizationAdmin(admin.ModelAdmin):
     list_select_related = (
         "job",
         "creator",
+        "invoice",
         "phase",
         "challenge",
         "algorithm_image",
@@ -109,6 +111,7 @@ class JobUtilizationAdmin(admin.ModelAdmin):
     )
     readonly_fields = (
         "creator",
+        "invoice",
         "phase",
         "challenge",
         "archive",
@@ -130,6 +133,7 @@ class JobWarmPoolUtilizationAdmin(admin.ModelAdmin):
         "creator",
         "duration",
         "compute_cost_euro_millicents",
+        "invoice",
         "phase",
         "challenge",
         "algorithm_image",
@@ -138,6 +142,7 @@ class JobWarmPoolUtilizationAdmin(admin.ModelAdmin):
     list_select_related = (
         "job",
         "creator",
+        "invoice",
         "phase",
         "challenge",
         "algorithm_image",
@@ -152,6 +157,7 @@ class JobWarmPoolUtilizationAdmin(admin.ModelAdmin):
     )
     readonly_fields = (
         "creator",
+        "invoice",
         "phase",
         "challenge",
         "archive",
@@ -175,6 +181,7 @@ class EvaluationUtilizationAdmin(admin.ModelAdmin):
         "creator",
         "duration",
         "compute_cost_euro_millicents",
+        "invoice",
         "phase",
         "challenge",
         "algorithm_image",
@@ -183,6 +190,7 @@ class EvaluationUtilizationAdmin(admin.ModelAdmin):
     list_select_related = (
         "evaluation",
         "creator",
+        "invoice",
         "phase",
         "challenge",
         "algorithm_image",
@@ -197,6 +205,7 @@ class EvaluationUtilizationAdmin(admin.ModelAdmin):
     )
     readonly_fields = (
         "creator",
+        "invoice",
         "phase",
         "challenge",
         "archive",