Speed up RankingData population

Author: epou

tigascoring/xp_scoring.py

@@ -610,38 +610,6 @@ def compute_user_score_in_xp_v2(user_uuid, update=False):
     return result
 
 
-def get_ranking_data( date_ini=None, date_end=datetime.datetime.today() ):
-    retval = {}
-    qs_reports = Report.objects.filter(creation_time__lte=datetime.datetime.today())
-    if date_ini is not None:
-        qs_reports = qs_reports.filter( creation_time__gte=date_ini )
-
-    uuid_replicas = get_uuid_replicas()
-    qs_overall = TigaUser.objects.exclude(score_v2=0).exclude(user_UUID__in=uuid_replicas)
-
-    overall_df = pd.DataFrame(list(qs_overall.values_list('score_v2', 'user_UUID')), columns=['score_v2', 'user_UUID'])
-    overall_sorted_df = overall_df.sort_values('score_v2', inplace=False)
-    overall_sorted_df["rank"] = overall_sorted_df['score_v2'].rank(method='dense', ascending=False)
-    overall_sorted_df.sort_values('rank', inplace=True)
-    min_max_overall = get_min_max(overall_sorted_df, 'score_v2')
-    min = min_max_overall['min']
-    max = min_max_overall['max']
-    retval['data'] = []
-    for index, row in overall_sorted_df.iterrows():
-        score = row['score_v2']
-        user_class = get_user_class( max, min, score)
-        retval['data'].append(
-            {
-                "score_v2": score,
-                "user_uuid":row['user_UUID'],
-                "identicon": '/media/identicons/' + row['user_UUID'] + '.png',
-                "class": user_class,
-                "rank": int(row['rank'])
-            }
-        )
-    return retval
-
-
 def compute_all_user_scores():
     all_users = TigaUser.objects.all()
     for user in all_users:

util_scripts/init_ranking_data.py

@@ -9,27 +9,49 @@
 os.chdir(proj_path)
 
 from django.core.wsgi import get_wsgi_application
+from django.db import transaction
+from django.db.models import Window, F, Min, Max
+from django.db.models.functions import DenseRank
+from django.utils import timezone
 
 application = get_wsgi_application()
 
-from tigascoring.xp_scoring import get_ranking_data
-from tigaserver_app.models import RankingData
-import datetime
+from tigascoring.xp_scoring import get_user_class
+from tigaserver_app.models import RankingData, TigaUser
 
+@transaction.atomic
 def init_ranking_data():
-    data = get_ranking_data()
+    # Step 1: Clear existing ranking data
     RankingData.objects.all().delete()
-    hydrated = []
-    for d in data['data']:
-        #hydrate models
-        r = RankingData(
-            user_uuid=d['user_uuid'],
-            class_value=d['class']['value'],
-            rank=d['rank'],
-            score_v2=d['score_v2']
+
+    # Step 2: Get ranked users from TigaUser
+    users_qs = TigaUser.objects.exclude(score_v2=0)
+    ranked_users = users_qs.annotate(
+        rank=Window(expression=DenseRank(), order_by=F('score_v2').desc())
+    ).order_by('-score_v2')
+
+    scores = users_qs.aggregate(
+        min_score=Min('score_v2'),
+        max_score=Max('score_v2')
+    )
+    min_score = scores['min_score']
+    max_score = scores['max_score']
+
+    # Step 3: Populate RankingData
+    last_update = timezone.now()
+    ranking_data_instances = []
+    for user in ranked_users.iterator():
+        ranking_data_instances.append(
+            RankingData(
+                user_uuid=user.pk,
+                class_value=get_user_class(min=min_score, max=max_score, user_score=user.score_v2)['value'],
+                rank=user.rank,
+                score_v2=user.score_v2,
+                last_update=last_update
+            )
         )
-        hydrated.append(r)
-    RankingData.objects.bulk_create(hydrated)
-    RankingData.objects.all().update(last_update=datetime.datetime.now())
+
+    # Bulk create for efficiency
+    RankingData.objects.bulk_create(ranking_data_instances)
 
 init_ranking_data()