Significantly improve leaderboard performance

Author: danlamanna

requirements.txt

@@ -51,6 +51,7 @@ markdown-it-py==3.0.0
 mdurl==0.1.2
 numpy==2.3.1
 oauthlib==3.2.2
+orjson==3.11.4
 packaging==25.0
 pandas==2.3.1
 pillow==10.4.0

setup.py

@@ -51,6 +51,7 @@
         'djangorestframework',
         'drf-yasg',
         'isic-challenge-scoring>=5.8',
+        'orjson',
         'requests',
         'rules',
         # See https://github.com/axnsan12/drf-yasg/issues/874

stade/core/templates/leaderboards.html

@@ -89,25 +89,25 @@ <h1 class="m-0 text-2xl font-normal text-gray-800">{{ challenge.name }} Leaderbo
               <tr class="cursor-pointer transition-colors duration-200 hover:bg-gray-100" @click="toggleDetails({{ submission.id }})" data-submission-id="{{ submission.id }}">
                 <td class="py-2 px-4 border-b border-gray-300">{{ forloop.counter }}</td>
                 <td class="py-2 px-4 border-b border-gray-300">
-                  <div>{{ submission.approach.team.name }}</div>
-                  {% if submission.approach.team.institution %}
+                  <div>{{ submission.approach__team__name }}</div>
+                  {% if submission.approach__team__institution %}
                     <div class="text-sm text-gray-600 mt-0.5">
-                      {{ submission.approach.team.institution }}
+                      {{ submission.approach__team__institution }}
                     </div>
                   {% endif %}
                 </td>
-                <td class="py-2 px-4 border-b border-gray-300">{{ submission.approach.name }}</td>
+                <td class="py-2 px-4 border-b border-gray-300">{{ submission.approach__name }}</td>
                 <td class="py-2 px-4 border-b border-gray-300">
-                  {% if submission.approach.manuscript_url %}
-                    <a href="{{ submission.approach.manuscript_url }}" target="_blank" title="View manuscript">
+                  {% if submission.approach__manuscript_url %}
+                    <a href="{{ submission.approach__manuscript_url }}" target="_blank" title="View manuscript">
                       <i class="material-icons text-green-600">description</i>
                     </a>
                   {% else %}
                     -
                   {% endif %}
                 </td>
                 <td class="py-2 px-4 border-b border-gray-300">
-                  {% if submission.approach.uses_external_data %}
+                  {% if submission.approach__uses_external_data %}
                     <div class="flex items-center gap-1">
                       <i class="material-icons text-orange-600">public</i>
                       <span>Yes</span>

stade/core/tests/test_leaderboard.py

@@ -81,18 +81,18 @@ def test_leaderboard_by_approach(task_with_submissions, client):
     submissions = resp.context['submissions']
     first, second, third, fourth = submissions[:4]
 
-    assert first.approach.team.name == 'team_0'
-    assert first.approach.name == 'approach_0'
-    assert first.overall_score == 0.95
-    assert second.approach.team.name == 'team_0'
-    assert second.approach.name == 'approach_1'
-    assert second.overall_score == 0.80
-    assert third.approach.team.name == 'team_1'
-    assert third.approach.name == 'approach_0'
-    assert third.overall_score == 0.78
-    assert fourth.approach.team.name == 'team_3'
-    assert fourth.approach.name == 'approach_1'
-    assert fourth.overall_score == 0.60
+    assert first['approach__team__name'] == 'team_0'
+    assert first['approach__name'] == 'approach_0'
+    assert first['overall_score'] == 0.95
+    assert second['approach__team__name'] == 'team_0'
+    assert second['approach__name'] == 'approach_1'
+    assert second['overall_score'] == 0.80
+    assert third['approach__team__name'] == 'team_1'
+    assert third['approach__name'] == 'approach_0'
+    assert third['overall_score'] == 0.78
+    assert fourth['approach__team__name'] == 'team_3'
+    assert fourth['approach__name'] == 'approach_1'
+    assert fourth['overall_score'] == 0.60
 
 
 @pytest.mark.django_db
@@ -109,12 +109,12 @@ def test_leaderboard_by_team(task_with_submissions, client):
     submissions = resp.context['submissions']
     first, second, third = submissions[:3]
 
-    assert first.approach.team.name == 'team_5'
-    assert first.approach.name == 'approach_0'
-    assert first.overall_score == 0.95
-    assert second.approach.team.name == 'team_6'
-    assert second.approach.name == 'approach_0'
-    assert second.overall_score == 0.78
-    assert third.approach.team.name == 'team_8'
-    assert third.approach.name == 'approach_1'
-    assert third.overall_score == 0.60
+    assert first['approach__team__name'] == 'team_5'
+    assert first['approach__name'] == 'approach_0'
+    assert first['overall_score'] == 0.95
+    assert second['approach__team__name'] == 'team_6'
+    assert second['approach__name'] == 'approach_0'
+    assert second['overall_score'] == 0.78
+    assert third['approach__team__name'] == 'team_8'
+    assert third['approach__name'] == 'approach_1'
+    assert third['overall_score'] == 0.60

stade/core/views.py

@@ -1,5 +1,4 @@
 from datetime import timedelta
-import json
 import logging
 
 from django.conf import settings
@@ -13,6 +12,7 @@
 from django.urls import reverse
 from django.utils import timezone
 from django.views.decorators.http import require_http_methods
+import orjson
 import requests
 from rules.contrib.views import objectgetter, permission_required
 
@@ -68,28 +68,47 @@ def leaderboard_page(request, challenge):
         submission_ids = list(submissions_by_approach(active_task.id))
 
     submissions = list(
-        Submission.objects.defer(None)  # avoid n+1 queries on the normally deferred score field
-        .select_related('approach', 'approach__team', 'creator')
-        .filter(id__in=submission_ids)
-        .order_by('-overall_score', 'created')
-    )[
-        0:200
-    ]  # leaderboards only show the top 200 results
+        Submission.objects.filter(id__in=submission_ids)
+        .values(
+            'id',
+            'overall_score',
+            'score',
+            'created',
+            'approach__name',
+            'approach__team__name',
+            'approach__team__institution',
+            'approach__uses_external_data',
+            'approach__manuscript',
+        )
+        .order_by('-overall_score', 'created')[:200]
+    )
 
     for submission in submissions:
-        if submission.score and isinstance(submission.score, dict):
-            submission.score_json = json.dumps(submission.score)
+        score = submission['score']
+        if score:
+            if isinstance(score, str):
+                submission['score_json'] = score
+            elif isinstance(score, dict):
+                submission['score_json'] = orjson.dumps(score).decode('utf-8')
+            else:
+                submission['score_json'] = '{}'
+        else:
+            submission['score_json'] = '{}'
+
+        if submission['approach__manuscript']:
+            from s3_file_field import S3FileField
+
+            field = S3FileField()
+            submission['approach__manuscript_url'] = field.storage.url(
+                submission['approach__manuscript']
+            )
         else:
-            submission.score_json = '{}'
+            submission['approach__manuscript_url'] = None
 
     stats = {
         'total_submissions': len(submissions),
-        'unique_teams': len(
-            set(s.approach.team.name for s in submissions if s.approach and s.approach.team)
-        ),
-        'used_external_data': sum(
-            1 for s in submissions if s.approach and s.approach.uses_external_data
-        ),
+        'unique_teams': len(set(s['approach__team__name'] for s in submissions)),
+        'used_external_data': sum(1 for s in submissions if s['approach__uses_external_data']),
     }
 
     return render(