Link ComponentJobUtilization and Invoice

app/grandchallenge/algorithms/models.py

@@ -1439,8 +1439,8 @@ def get_or_create_display_set(self, *, reader_study):
 
         return display_set
 
-    def create_utilization(self):
-        JobUtilization.objects.create(job=self)
+    def create_utilization(self, *, invoice):
+        JobUtilization.objects.create(job=self, invoice=invoice)
 
     @property
     def utilization(self):

app/grandchallenge/algorithms/tasks.py

@@ -10,6 +10,7 @@
 from django.utils import timezone
 
 from grandchallenge.algorithms.exceptions import TooManyJobsScheduled
+from grandchallenge.challenges.exceptions import InsufficientBudgetError
 from grandchallenge.components.schemas import GPUTypeChoices
 from grandchallenge.components.tasks import (
     remove_container_image_from_registry,
@@ -164,6 +165,21 @@ def create_algorithm_jobs(
                 input_civ_set=ai.values.all(),
                 use_warm_pool=use_warm_pool,
             )
+            jobs.append(
+                job
+            )  # Keep track of created jobs, even if utilization setup has failed
+
+            if job_utilization_challenge is not None:
+                try:
+                    job.utilization.invoice = (
+                        job_utilization_challenge.active_invoice
+                    )
+                except InsufficientBudgetError:
+                    job.update_status(
+                        status=Job.FAILURE,
+                        error_message="Job cannot be executed. The challenge has insufficient budget to run this job.",
+                    )
+                    break
 
             job.utilization.archive = ai.archive
             job.utilization.phase = job_utilization_phase
@@ -172,8 +188,6 @@ def create_algorithm_jobs(
 
             job.execute()
 
-            jobs.append(job)
-
     return jobs
 
 

app/grandchallenge/challenges/costs.py

@@ -1,5 +1,6 @@
 from django.contrib.auth.models import Permission
-from django.db.models import Sum
+from django.db.models import Sum, Value
+from django.db.models.functions import Coalesce
 
 from grandchallenge.algorithms.models import (
     AlgorithmImage,
@@ -13,30 +14,30 @@
     EvaluationGroundTruth,
     Method,
 )
+from grandchallenge.invoices.models import Invoice
 from grandchallenge.utilization.models import (
     EvaluationUtilization,
     JobUtilization,
     JobWarmPoolUtilization,
 )
 
 
-def annotate_compute_costs(*, challenge):
-    algorithm_job_utilizations = JobUtilization.objects.filter(
-        challenge=challenge
-    )
-    job_warm_pool_utilizations = JobWarmPoolUtilization.objects.filter(
-        challenge=challenge
-    )
-    evaluation_job_utilizations = EvaluationUtilization.objects.filter(
-        challenge=challenge
-    )
-
-    update_compute_cost_euro_millicents(
-        obj=challenge,
-        algorithm_job_utilizations=algorithm_job_utilizations,
-        job_warm_pool_utilizations=job_warm_pool_utilizations,
-        evaluation_job_utilizations=evaluation_job_utilizations,
+def annotate_compute_costs(*, invoice):
+    result = Invoice.objects.filter(pk=invoice.pk).aggregate(
+        total=Coalesce(
+            Sum("job_utilizations__compute_cost_euro_millicents"),
+            Value(0),
+        )
+        + Coalesce(
+            Sum("job_warm_pool_utilizations__compute_cost_euro_millicents"),
+            Value(0),
+        )
+        + Coalesce(
+            Sum("evaluation_utilizations__compute_cost_euro_millicents"),
+            Value(0),
+        )
     )
+    invoice.compute_costs_utilized_euros_millicents = result["total"]
 
 
 def annotate_job_duration_and_compute_costs(*, phase):

app/grandchallenge/challenges/exceptions.py

@@ -0,0 +1,2 @@
+class InsufficientBudgetError(Exception):
+    pass

app/grandchallenge/challenges/models.py

@@ -43,6 +43,7 @@
     send_challenge_requested_email_to_reviewers,
     send_email_percent_budget_consumed_alert,
 )
+from grandchallenge.challenges.exceptions import InsufficientBudgetError
 from grandchallenge.challenges.utils import ChallengeTypeChoices
 from grandchallenge.components.models import APIMethodChoices
 from grandchallenge.components.schemas import GPUTypeChoices
@@ -937,6 +938,37 @@ def has_paid_prepaid_invoice(self):
             payment_status=PaymentStatusChoices.PAID,
         ).exists()
 
+    @cached_property
+    def compute_costs_balance_euros_millicents(self):
+        return self._get_compute_costs_balance_euros_millicents(
+            invoices=self.invoices.all()
+        )
+
+    def _get_compute_costs_balance_euros_millicents(self, *, invoices):
+        return sum(
+            invoice.compute_costs_balance_euros_millicents
+            for invoice in invoices
+        )
+
+    @property
+    def active_invoice(self):
+        invoices = self.invoices.order_by("expires_on", "created")
+
+        if (
+            self._get_compute_costs_balance_euros_millicents(invoices=invoices)
+            <= 0
+        ):
+            raise InsufficientBudgetError
+
+        for invoice in invoices:
+            if invoice.compute_costs_balance_euros_millicents > 0:
+                return invoice
+
+        # Defensive catch
+        raise ValueError(
+            "Unexpected: this challenge has no available budget in any invoice despite a positive total balance."
+        )
+
 
 class ChallengeUserObjectPermission(UserObjectPermissionBase):
     allowed_permissions = frozenset()

app/grandchallenge/challenges/tasks.py

@@ -6,7 +6,7 @@
 from django.conf import settings
 from django.contrib.auth import get_user_model
 from django.db import transaction
-from django.db.models import Count, Max, Min, Q
+from django.db.models import Count, Max, Min, Q, Sum
 from django.utils.timezone import datetime, now
 from psycopg.errors import LockNotAvailable
 
@@ -31,6 +31,7 @@
     acks_late_micro_short_task,
 )
 from grandchallenge.evaluation.models import Evaluation, Phase
+from grandchallenge.invoices.models import Invoice
 
 
 @acks_late_2xlarge_task
@@ -78,7 +79,13 @@ def update_challenge_results_cache():
     )
 
 
-def retry_with_backoff(exceptions, max_attempts=5, base_delay=1, max_delay=10):
+def retry_with_backoff(
+    *,
+    exceptions,
+    max_attempts=5,
+    base_delay=1,
+    max_delay=10,
+):
     def decorator(func):
         @functools.wraps(func)
         def wrapper(*args, **kwargs):
@@ -102,23 +109,34 @@ def wrapper(*args, **kwargs):
 
 @acks_late_2xlarge_task
 def update_challenge_compute_costs():
-    for challenge in Challenge.objects.with_available_compute().iterator(
-        chunk_size=1000
-    ):
+    for invoice in Invoice.objects.iterator(chunk_size=1000):
         with transaction.atomic():
-            annotate_compute_costs(challenge=challenge)
+            annotate_compute_costs(invoice=invoice)
 
-            @retry_with_backoff((LockNotAvailable,))
-            def save_challenge():
+            @retry_with_backoff(exceptions=(LockNotAvailable,))
+            def save_invoice():
+                invoice.save(
+                    update_fields=("compute_costs_utilized_euros_millicents",)
+                )
+
+                # TODO: remove this once compute_cost_euro_millicents is removed or deprecated
+                challenge = Challenge.objects.with_available_compute().get(
+                    pk=invoice.challenge_id
+                )
+                challenge.compute_cost_euro_millicents = (
+                    Invoice.objects.filter(challenge=challenge.pk).aggregate(
+                        total=Sum("compute_costs_utilized_euros_millicents")
+                    )["total"]
+                )
                 challenge.save(update_fields=("compute_cost_euro_millicents",))
 
-            save_challenge()
+            save_invoice()
 
     for phase in Phase.objects.iterator(chunk_size=1000):
         with transaction.atomic():
             annotate_job_duration_and_compute_costs(phase=phase)
 
-            @retry_with_backoff((LockNotAvailable,))
+            @retry_with_backoff(exceptions=(LockNotAvailable,))
             def save_phase():
                 phase.save(
                     skip_calculate_ranks=True,
@@ -137,7 +155,7 @@ def update_challenge_storage_size():
         with transaction.atomic():
             annotate_storage_size(challenge=challenge)
 
-            @retry_with_backoff((LockNotAvailable,))
+            @retry_with_backoff(exceptions=(LockNotAvailable,))
             def save_challenge():
                 challenge.save(
                     update_fields=(

app/grandchallenge/components/models.py

@@ -1661,6 +1661,11 @@ def retrieve_existing_civs(*, civ_data_objects):
 
         return existing_civs
 
+    def create(self, *args, utilization_invoice=None, **kwargs):
+        component_job = super().create(*args, **kwargs)
+        component_job.create_utilization(invoice=utilization_invoice)
+        return component_job
+
 
 class ComponentJob(FieldChangeMixin, UUIDModel):
     # The job statuses come directly from celery.result.AsyncResult.status:
@@ -1818,9 +1823,6 @@ def save(self, *args, **kwargs):
 
         super().save()
 
-        if adding:
-            self.create_utilization()
-
     def update_status(  # noqa:C901
         self,
         *,
@@ -2032,7 +2034,7 @@ def runtime_metrics_chart(self):
             ],
         )
 
-    def create_utilization(self):
+    def create_utilization(self, *, invoice):
         raise NotImplementedError
 
     @property

app/grandchallenge/evaluation/admin.py

@@ -3,6 +3,7 @@
 from django.forms import ModelForm
 from django.utils.html import format_html
 
+from grandchallenge.challenges.exceptions import InsufficientBudgetError
 from grandchallenge.components.admin import (
     ComponentImageAdmin,
     cancel_jobs,
@@ -130,7 +131,19 @@ def reevaluate_submissions(modeladmin, request, queryset):
                 messages.WARNING,
             )
         else:
-            submission.create_evaluation(additional_inputs=None)
+            try:
+                invoice = submission.phase.challenge.active_invoice
+            except InsufficientBudgetError:
+                modeladmin.message_user(
+                    request,
+                    f"Submission {submission.pk} cannot be reevaluated. Challenge has insufficient budget.",
+                    messages.WARNING,
+                )
+            else:
+                submission.create_evaluation(
+                    additional_inputs=None,
+                    invoice=invoice,
+                )
 
 
 @admin.register(Submission)

app/grandchallenge/evaluation/forms.py

@@ -24,7 +24,7 @@
 
 from grandchallenge.algorithms.forms import UserAlgorithmsForPhaseMixin
 from grandchallenge.algorithms.models import Job
-from grandchallenge.challenges.models import Challenge
+from grandchallenge.challenges.exceptions import InsufficientBudgetError
 from grandchallenge.components.forms import (
     AdditionalInputsMixin,
     ContainerImageForm,
@@ -471,6 +471,15 @@ def clean(self):
                 "You must confirm that you want to submit to this phase."
             )
 
+        try:
+            invoice = self._phase.challenge.active_invoice
+        except InsufficientBudgetError:
+            raise ValidationError(
+                "Challenge has insufficient budget. Please contact the challenge organizers."
+            )
+        else:
+            cleaned_data["invoice"] = invoice
+
         return cleaned_data
 
     def clean_phase(self):
@@ -655,7 +664,8 @@ def save(self, *args, **kwargs):
         instance = super().save(*args, **kwargs)
 
         instance.create_evaluation(
-            additional_inputs=self.cleaned_data["additional_inputs"]
+            additional_inputs=self.cleaned_data["additional_inputs"],
+            invoice=self.cleaned_data["invoice"],
         )
 
         return instance
@@ -741,17 +751,14 @@ def clean(self):
                     "Please wait for the other evaluation to complete."
                 )
 
-        # Fetch from the db to get the cost annotations
-        challenge = (
-            Challenge.objects.filter(
-                pk=cleaned_data["submission"].phase.challenge.pk
+        try:
+            invoice = cleaned_data["submission"].phase.challenge.active_invoice
+        except InsufficientBudgetError:
+            raise ValidationError(
+                "Challenge has insufficient budget. Please contact the challenge organizers."
             )
-            .with_available_compute()
-            .get()
-        )
-
-        if challenge.available_compute_euro_millicents <= 0:
-            raise ValidationError("This challenge has exceeded its budget")
+        else:
+            cleaned_data["invoice"] = invoice
 
         if Evaluation.objects.get_evaluations_with_same_inputs(
             inputs=cleaned_data["additional_inputs"],

app/grandchallenge/evaluation/models.py

@@ -1651,7 +1651,7 @@ def save(self, *args, **kwargs):
                     send_action=False,
                 )
 
-    def create_evaluation(self, *, additional_inputs):
+    def create_evaluation(self, *, additional_inputs, invoice):
         if (
             self.phase.additional_evaluation_inputs.exists()
             and not additional_inputs
@@ -1711,6 +1711,7 @@ def create_evaluation(self, *, additional_inputs):
                 requires_gpu_type=self.phase.evaluation_requires_gpu_type,
                 requires_memory_gb=self.phase.evaluation_requires_memory_gb,
                 status=Evaluation.VALIDATING_INPUTS,
+                utilization_invoice=invoice,
             )
 
         if self.phase.submission_kind == SubmissionKindChoices.ALGORITHM:
@@ -2459,8 +2460,8 @@ def status_url(self) -> str:
             },
         )
 
-    def create_utilization(self):
-        EvaluationUtilization.objects.create(evaluation=self)
+    def create_utilization(self, *, invoice):
+        EvaluationUtilization.objects.create(evaluation=self, invoice=invoice)
 
     @property
     def utilization(self):

app/grandchallenge/evaluation/views/__init__.py

@@ -535,7 +535,8 @@ def get_success_url(self):
     def form_valid(self, form):
         redirect = super().form_valid(form)
         self.submission.create_evaluation(
-            additional_inputs=form.cleaned_data["additional_inputs"]
+            additional_inputs=form.cleaned_data["additional_inputs"],
+            invoice=form.cleaned_data["invoice"],
         )
         return redirect