Added TigaUser.update_score

Author: epou

requirements/prod_20_04.pip

@@ -61,6 +61,7 @@ MarkupSafe==1.1.1
 matplotlib==2.0.2
 msgpack==1.0.2
 numpy==1.23.1
+numpyencoder==0.3.0
 odfpy==1.4.0
 openapi-codec==1.3.2
 openpyxl==3.0.0

stats/views.py

@@ -19,13 +19,11 @@
 import json
 import datetime
 from django.utils import timezone
-from tigascoring.xp_scoring import compute_user_score_in_xp_v2
 from rest_framework.exceptions import ParseError
 from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
 import math
 from django.utils import translation
 
-from tigascoring.xp_scoring import compute_user_score_in_xp_v2
 from rest_framework.exceptions import ParseError
 from django.core.paginator import Paginator
 import math
@@ -642,13 +640,8 @@ def stats_user_score(request, user_uuid=None):
             pass
 
     if user.score_v2_struct is None:
-        user_score = compute_user_score_in_xp_v2(user_uuid, update=True)
-        user.score_v2_struct = json.dumps(user_score, indent=2, sort_keys=True, default=str)
-        user.last_score_update = datetime.datetime.now()
-        user.save()
-    else:
-        user_score = json.loads(user.score_v2_struct)
-    context = { "score_data": user_score, "score_last_update": user.last_score_update }
+        user.update_score(commit=True)
+    context = { "score_data": user.score_v2_struct, "score_last_update": user.last_score_update }
     return render(request, 'stats/user_score.html', context)
 
 
@@ -680,17 +673,11 @@ def stats_user_ranking(request, page=1, user_uuid=None):
         try:
             user = TigaUser.objects.get(pk=user_uuid)
             user.get_identicon()
-            #user_score = compute_user_score_in_xp_v2(user_uuid, update=True)
-            if user.score_v2_struct is None:
-                user_score = compute_user_score_in_xp_v2(user_uuid, update=True)
-                user.score_v2_struct = json.dumps(user_score, indent=2, sort_keys=True, default=str)
-                user.last_score_update = datetime.datetime.now()
-                user.save()
-            else:
-                user_score = json.loads( user.score_v2_struct )
-            if user_score['total_score'] > 0:
-                user_has_score = True
 
+            if user.score_v2_struct is None:
+                user.update_score(commit=True)
+            user_score = user.score_v2_struct
+            user_has_score = bool(user.score_v2)
         except TigaUser.DoesNotExist:
             pass
     seek = request.GET.get('seek', 'f')
@@ -1046,13 +1033,9 @@ def get_user_xp_data(request):
     translation.activate(locale)
 
     if u.score_v2_struct is None:
-        retval = compute_user_score_in_xp_v2(user_id, update=True)
-        u.score_v2_struct = json.dumps(retval, indent=2, sort_keys=True, default=str)
-        u.last_score_update = datetime.datetime.now()
-        u.save()
-    else:
-        retval = json.loads(u.score_v2_struct)
+        u.update_score(commit=True)
 
+    retval = u.score_v2_struct
     retval['last_update'] = u.last_score_update
 
     return Response(retval)

tigascoring/xp_scoring.py

@@ -1,4 +1,4 @@
-from tigaserver_app.models import Report, TigaUser, TigaProfile, Award
+from tigaserver_app.models import Report, TigaUser, Award
 from tigaserver_project import settings as conf
 import pandas as pd
 import datetime
@@ -324,106 +324,17 @@ def get_unrelated_awards_score( user_uuid, user_uuids ):
 #         return str(diff.days) + _(" days ago")
 
 
-def compute_user_score_in_xp_v2_fast(user_uuid):
+def compute_user_score_in_xp_v2(user_uuid):
 
     user = TigaUser.objects.get(pk=user_uuid)
     user_uuids = None
     if user.profile is not None:
         user_uuids = TigaUser.objects.filter(profile=user.profile).values('user_UUID')
 
-    result = {}
-    result['total_score'] = 0
-    result['user_uuid'] = user_uuid
-    result['score_detail'] = {}
-
-    if user_uuids is None:
-        user_reports = Report.objects.filter(user__user_UUID=user_uuid).order_by('-creation_time')
-    else:
-        user_reports = Report.objects.filter(user__user_UUID__in=user_uuids).order_by('-creation_time')
-
-    adults = user_reports.filter(type='adult')
-    #bites = user_reports.filter(type='bite')
-    sites = user_reports.filter(type='site')
-
-    adult_last_versions = adults.non_deleted().all()
-    #bite_last_versions = bites.non_deleted().all()
-    site_last_versions = sites.non_deleted().all()
-
-    results_adult = {}
-    results_adult['score'] = 0
-    results_adult['score_items'] = []
-    result['score_detail']['adult'] = results_adult
-
-    adult_score = 0
-    for report in adult_last_versions:
-        result = get_adult_report_score(report, result)
-        index = len(result['score_detail']['adult']['score_items']) - 1
-        result['score_detail']['adult']['score'] += result['score_detail']['adult']['score_items'][index][
-            'report_score']
-        adult_score += result['score_detail']['adult']['score_items'][index]['report_score']
-    result['total_score'] += adult_score
-
-    '''
-    results_bite = {}
-    results_bite['score'] = 0
-    results_bite['score_items'] = []
-    result['score_detail']['bite'] = results_bite
-
-    bite_score = 0
-    for report in bite_last_versions:
-        result = get_bite_report_score(report, result)
-        index = len(result['score_detail']['bite']['score_items']) - 1
-        result['score_detail']['bite']['score'] += result['score_detail']['bite']['score_items'][index]['report_score']
-        bite_score += result['score_detail']['bite']['score_items'][index]['report_score']
-    result['total_score'] += bite_score
-    '''
-
-    results_site = {}
-    results_site['score'] = 0
-    results_site['score_items'] = []
-    result['score_detail']['site'] = results_site
-
-    site_score = 0
-    for report in site_last_versions:
-        result = get_site_report_score(report, result)
-        index = len(result['score_detail']['site']['score_items']) - 1
-        result['score_detail']['site']['score'] += result['score_detail']['site']['score_items'][index]['report_score']
-        site_score += result['score_detail']['site']['score_items'][index]['report_score']
-    result['total_score'] += site_score
-
-    unrelated_score = get_unrelated_awards_score(user_uuid, user_uuids)
-
-    result['total_score'] += unrelated_score['score']
-
-    return result
-
-
-def get_uuid_replicas():
-    profiles = TigaProfile.objects.all()
-    exclude = []
-    for p in profiles:
-        if p.profile_devices.count() > 1:
-            i = 0
-            for d in p.profile_devices.all().order_by('user_UUID'):
-                if i > 0:
-                    exclude.append(d.user_UUID)
-                i+=1
-    return exclude
-
-
-def compute_user_score_in_xp_v2(user_uuid, update=False):
-
-    user = TigaUser.objects.get(pk=user_uuid)
-    user_uuids = None
-    if user.profile is not None:
-        user_uuids = TigaUser.objects.filter(profile=user.profile).values('user_UUID')
-
-    uuid_replicas = get_uuid_replicas()
-
-    qs_overall = TigaUser.objects.exclude(score_v2=0).exclude(user_UUID__in=uuid_replicas)
-    qs_adult = TigaUser.objects.exclude(score_v2_adult=0).exclude(user_UUID__in=uuid_replicas)
-    qs_site = TigaUser.objects.exclude(score_v2_site=0).exclude(user_UUID__in=uuid_replicas)
-    #qs_bite = TigaUser.objects.exclude(score_v2_bite=0).exclude(user_UUID__in=uuid_replicas)
+    qs_overall = TigaUser.objects.exclude(score_v2=0)
+    qs_adult = TigaUser.objects.exclude(score_v2_adult=0)
+    qs_site = TigaUser.objects.exclude(score_v2_site=0)
+    #qs_bite = TigaUser.objects.exclude(score_v2_bite=0)
 
     overall_df = pd.DataFrame(list(qs_overall.values_list('score_v2', 'user_UUID')), columns=['score_v2', 'user_UUID'])
     adult_df = pd.DataFrame(list(qs_adult.values_list('score_v2_adult', 'user_UUID')), columns=['score_v2_adult', 'user_UUID'])
@@ -587,18 +498,6 @@ def compute_user_score_in_xp_v2(user_uuid, update=False):
         result['score_detail']['site']['top_perc'] = (float(site_number_below_rank) / float(site_number_total)) * 100.0
     result['score_detail']['site']['ranked_users'] = site_number_total
 
-    if update:
-        if user_uuids is not None:
-            all_users_in_profile = TigaUser.objects.filter(user_UUID__in=user_uuids)
-            all_users_in_profile.update(score_v2=result['total_score'])
-            all_users_in_profile.update(score_v2_adult=result['score_detail']['adult']['score'])
-            all_users_in_profile.update(score_v2_site=result['score_detail']['site']['score'])
-        else:
-            user.score_v2 = result['total_score']
-            user.score_v2_adult = result['score_detail']['adult']['score']
-            user.score_v2_site = result['score_detail']['site']['score']
-            user.save()
-
     '''
     if bite_number_below_rank == 0 and bite_number_total == 0:
         result['score_detail']['bite']['top_perc'] = 100.0
@@ -616,8 +515,7 @@ def get_ranking_data( date_ini=None, date_end=datetime.datetime.today() ):
     if date_ini is not None:
         qs_reports = qs_reports.filter( creation_time__gte=date_ini )
 
-    uuid_replicas = get_uuid_replicas()
-    qs_overall = TigaUser.objects.exclude(score_v2=0).exclude(user_UUID__in=uuid_replicas)
+    qs_overall = TigaUser.objects.exclude(score_v2=0)
 
     overall_df = pd.DataFrame(list(qs_overall.values_list('score_v2', 'user_UUID')), columns=['score_v2', 'user_UUID'])
     overall_sorted_df = overall_df.sort_values('score_v2', inplace=False)
@@ -642,14 +540,6 @@ def get_ranking_data( date_ini=None, date_end=datetime.datetime.today() ):
     return retval
 
 
-def compute_all_user_scores():
-    all_users = TigaUser.objects.all()
-    for user in all_users:
-        score = compute_user_score_in_xp_v2( user.user_UUID )
-        user.score_v2 = score
-        user.save()
-
-
 def get_all_user_reports(user_uuid):
     user = TigaUser.objects.get(pk=user_uuid)
     user_uuids = None

tigaserver_app/migrations/0053_tigauser_update_score_jsonfield.py

@@ -0,0 +1,19 @@
+# Generated by Django 2.2.7 on 2024-03-15 11:50
+
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations
+import numpyencoder.numpyencoder
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('tigaserver_app', '0052_ensure_masked_fixes'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='tigauser',
+            name='score_v2_struct',
+            field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, encoder=numpyencoder.numpyencoder.NumpyEncoder, help_text='Full cached score data', null=True),
+        ),
+    ]

tigaserver_app/models.py

@@ -8,6 +8,7 @@
 from firebase_admin.messaging import Message, Notification as FirebaseNotification, AndroidConfig, AndroidNotification
 import logging
 from math import floor
+from numpyencoder import NumpyEncoder
 from PIL import Image
 import pydenticon
 import os
@@ -21,16 +22,15 @@
 from django.contrib.gis.db.models.functions import Distance as DistanceFunction
 from django.contrib.gis.geos import GEOSGeometry
 from django.contrib.gis.measure import Distance as DistanceMeasure
+from django.contrib.postgres.fields import JSONField
 from django.db.models import Count, Q
 from django.db.models.signals import post_save
 from django.dispatch import receiver
 from django.forms.models import model_to_dict
 from django.template.loader import render_to_string, TemplateDoesNotExist
 from django.urls import reverse
-from django.utils import translation
+from django.utils import translation, timezone
 from django.utils.deconstruct import deconstructible
-from django.utils import timezone
-from django.utils.timezone import utc
 from django.utils.translation import ugettext_lazy as _
 
 from simple_history.models import HistoricalRecords
@@ -222,7 +222,9 @@ class TigaUser(models.Model):
 
     profile = models.ForeignKey(TigaProfile, related_name='profile_devices', null=True, blank=True, on_delete=models.SET_NULL, )
 
-    score_v2_struct = models.TextField(help_text="Full cached score data", null=True, blank=True)
+    # NOTE using NumpyEncoder since compute_user_score_in_xp_v2 function get result from pandas dataframe
+    # and some integer are np.int64, which can not be encoded with the regular json library setup.
+    score_v2_struct = JSONField(encoder=NumpyEncoder, help_text="Full cached score data", null=True, blank=True)
 
     last_score_update = models.DateTimeField(help_text="Last time score was updated", null=True, blank=True)
 
@@ -251,6 +253,34 @@ def get_identicon(self):
             f.close()
         return settings.MEDIA_URL + "identicons/" + self.user_UUID + ".png"
 
+    def update_score(self, commit: bool = True) -> None:
+        # NOTE: placing import here due to circular import
+        from tigascoring.xp_scoring import compute_user_score_in_xp_v2
+
+        score_dict = compute_user_score_in_xp_v2(user_uuid=self.pk)
+        self.score_v2_struct = score_dict
+
+        try:
+            self.score_v2_adult = score_dict['score_detail']['adult']['score']
+        except (KeyError, TypeError):
+            self.score_v2_adult = 0
+
+        try:
+            self.score_v2_bite = score_dict['score_detail']['bite']['score']
+        except (KeyError, TypeError):
+            self.score_v2_bite = 0
+
+        try:
+            self.score_v2_site = score_dict['score_detail']['site']['score']
+        except (KeyError, TypeError):
+            self.score_v2_site = 0
+
+        self.score_v2 = sum([self.score_v2_adult, self.score_v2_bite, self.score_v2_site])
+        self.last_score_update = timezone.now()
+
+        if commit:
+            self.save()
+
     n_reports = property(number_of_reports_uploaded)
     ios_user = property(is_ios)
 
@@ -345,7 +375,7 @@ def __unicode__(self):
         return self.title_catalan
 
     def active_missions(self):
-        return self.expiration_time >= datetime.utcnow().replace(tzinfo=utc)
+        return self.expiration_time >= datetime.utcnow().replace(tzinfo=timezone.utc)
 
 
 class MissionTrigger(models.Model):

tigaserver_app/views.py

@@ -32,7 +32,6 @@
 from django.views.decorators.cache import cache_page
 from tigacrafting.criteria import users_with_pictures,users_with_storm_drain_pictures, users_with_score, users_with_score_range, users_with_topic
 from tigascoring.maUsers import smmry
-from tigascoring.xp_scoring import compute_user_score_in_xp_v2
 from tigaserver_app.serializers import custom_render_notification,score_label
 import tigaserver_project.settings as conf
 import copy
@@ -1219,8 +1218,8 @@ def user_score_v2(request):
     if user_id == -1:
         raise ParseError(detail='user_id is mandatory')
     user = get_object_or_404(TigaUser.objects.all(), pk=user_id)
-    result = compute_user_score_in_xp_v2(user_id, update=True)
-    return Response(result)
+    user.update_score(commit=True)
+    return Response(user.score_v2_struct)
 
 
 @api_view(['GET', 'POST'])