Merge pull request #201 from Mosquito-Alert/report_versioning

Report versioning

Author: epou

requirements/prod_20_04.pip

@@ -27,6 +27,7 @@ django-import-export==1.2.0
 django-leaflet==0.13.2
 django-messages==0.6.0
 django-rest-swagger==2.2.0
+django-simple-history==3.0.0
 django-sslserver==0.22
 django-taggit==1.1.0
 djangorestframework==3.10.3

stats/views.py

@@ -1002,7 +1002,7 @@ def show_fix_users(request):
 
 
 def show_report_users(request):
-    real_reports = [report for report in Report.objects.filter(Q(package_name='Tigatrapp', creation_time__gte=date(2014, 6, 24)) | Q(package_name='ceab.movelab.tigatrapp', package_version__gt=3)) if report.latest_version]
+    real_reports = [report for report in Report.objects.filter(Q(package_name='Tigatrapp', creation_time__gte=date(2014, 6, 24)) | Q(package_name='ceab.movelab.tigatrapp', package_version__gt=3))]
     tz = get_localzone()
     ref_date = datetime.datetime(2014, 6, 13,  tzinfo=tz)
     end_date = tz.localize(datetime.datetime.now())
@@ -1073,10 +1073,9 @@ def get_hashtag_map_data(request):
         r = Report.objects.filter(note__icontains=hashtag).order_by('-server_upload_time')[:200]
         n = 0
         for report in r:
-            if report.latest_version:
-                n = n + 1
-                dates.append(report.server_upload_time)
-                data.append({ 'note': report.note, 'picture': report.photo_html, 'lat': report.lat, 'lon': report.lon, 'date_uploaded': report.server_upload_time.strftime('%d-%m-%Y / %H:%M:%S') })
+            n = n + 1
+            dates.append(report.server_upload_time)
+            data.append({ 'note': report.note, 'picture': report.photo_html, 'lat': report.lat, 'lon': report.lon, 'date_uploaded': report.server_upload_time.strftime('%d-%m-%Y / %H:%M:%S') })
     min_date_str = ''
     max_date_str = ''
     if len(dates) == 0:

tigacrafting/criteria.py

@@ -12,12 +12,12 @@ def filter_users_by_score(score):
     return score_user_ids
 
 def filter_users_with_storm_drain_pictures(reports):
-    reports_filtered = filter(lambda x: not x.deleted and x.latest_version and x.embornals, reports)
+    reports_filtered = reports.non_deleted()
+    reports_filtered = filter(lambda x: x.embornals, reports_filtered)
     return reports_filtered
 
 def filter_users_with_pictures(reports):
-    reports_filtered = filter(lambda x: not x.deleted and x.latest_version, reports)
-    return reports_filtered
+    return list(reports.non_deleted().all())
 
 def users_with_pictures():
     return filter_reports('users_with_pictures')

tigacrafting/migrations/0020_auto_20240125_1557.py

@@ -0,0 +1,17 @@
+# Generated by Django 2.2.7 on 2024-01-25 15:57
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('tigacrafting', '0019_auto_20240122_1709'),
+    ]
+
+    operations = [
+        migrations.RemoveConstraint(
+            model_name='expertreportannotation',
+            name='unique_assignation',
+        )
+    ]

tigacrafting/migrations/0021_auto_20240125_1557.py

@@ -0,0 +1,32 @@
+# Generated by Django 2.2.7 on 2024-01-25 15:57
+
+from django.db import migrations, models
+from django.db.models import OuterRef, Subquery
+
+def remove_expert_report_annotation_duplicates(apps, schema_editor):
+    ExpertReportAnnotation = apps.get_model("tigacrafting", "ExpertReportAnnotation")
+
+    # Will keep only the one with the most recent last_modified date.
+    last_annotation_subquery = ExpertReportAnnotation.objects.filter(
+        user=OuterRef('user'),
+        report=OuterRef('report')
+    ).order_by('-last_modified').values('pk')[:1]
+
+    print("Deleting duplicate ExpertReportAnnotation, keeping last modified...")
+    print(ExpertReportAnnotation.objects.exclude(pk__in=Subquery(last_annotation_subquery)).delete())
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('tigacrafting', '0020_auto_20240125_1557'),
+        ('tigaserver_app', '0050_report_versioning'),
+    ]
+
+    operations = [
+        migrations.RunPython(remove_expert_report_annotation_duplicates, migrations.RunPython.noop),
+        migrations.AddConstraint(
+            model_name='expertreportannotation',
+            constraint=models.UniqueConstraint(fields=('user', 'report'), name='unique_assignation'),
+        ),
+    ]

tigacrafting/report_queues.py

@@ -1,7 +1,6 @@
 from tigaserver_app.models import Report, ExpertReportAnnotation, EuropeCountry
 from django.core.exceptions import ObjectDoesNotExist
 from tigacrafting.models import UserStat
-from django.db.models.expressions import RawSQL
 from django.contrib.auth.models import User
 from django.db.models import Count
 from django.db.models import Q
@@ -56,28 +55,28 @@ def filter_reports_for_superexpert(reports):
     # not deleted, last version, completely validated by at least three experts
     #deleted_adult_reports = get_deleted_adult_reports(reports)
     # not deleted
-    undeleted = reports.exclude(version_UUID__in=RawSQL("select \"version_UUID\" from tigaserver_app_report r, (select report_id, user_id, count(\"version_UUID\") from tigaserver_app_report where type = 'adult' and report_id in (select distinct report_id from tigaserver_app_report where version_number = -1) group by report_id, user_id having count(\"version_UUID\") >1) as deleted where r.report_id = deleted.report_id and r.user_id = deleted.user_id",()))
-    # last version
-    latest_versions = undeleted.filter(version_UUID__in=RawSQL("select \"version_UUID\" from tigaserver_app_report r,(select report_id,max(version_number) as higher from tigaserver_app_report where type = 'adult' group by report_id) maxes where r.type = 'adult' and r.report_id = maxes.report_id and r.version_number = maxes.higher",()))
-    #latest_versions = undeleted.filter(version_UUID__in=RawSQL("select \"version_UUID\" from tigaserver_app_report r,(select report_id,max(version_number) as higher from tigaserver_app_report where type = 'adult' group by report_id) maxes where r.type = 'adult' and r.report_id = maxes.report_id and r.version_number = maxes.higher union select \"version_UUID\" from tigaserver_app_report r, (select report_id, user_id, version_number, count(\"version_UUID\") from tigaserver_app_report where type = 'adult' group by report_id, user_id, version_number having count(\"version_UUID\") = 1) as uniq where r.report_id = uniq.report_id and r.user_id = uniq.user_id and r.version_number = uniq.version_number",()))
+    reports_qs = reports.filter(type=Report.TYPE_ADULT).non_deleted()
     # fully validated
     experts = User.objects.filter(groups__name='expert')
-    fully_validated = ExpertReportAnnotation.objects.filter(report__in=latest_versions).filter(user__in=experts).filter(validation_complete=True).values('report').annotate(n_validations=Count('report')).filter(n_validations__gte=MAX_N_OF_EXPERTS_ASSIGNED_PER_REPORT)
+    fully_validated = ExpertReportAnnotation.objects.filter(
+        report__in=reports_qs,
+        user__in=experts,
+        validation_complete=True
+    ).values('report').annotate(
+        n_validations=Count('report')
+    ).filter(
+        n_validations__gte=MAX_N_OF_EXPERTS_ASSIGNED_PER_REPORT
+    )
+
     report_ids = set(fully_validated.values_list('report', flat=True))
-    filtered_reports = latest_versions.filter(version_UUID__in=report_ids)
+    filtered_reports = reports_qs.filter(version_UUID__in=report_ids)
     #filtered_reports = filter(lambda x: len(list(filter(lambda y: y.is_expert() and y.validation_complete, x.expert_report_annotations.all()))) >= 3, latest_versions)
     return filtered_reports
 
 
 def filter_reports(reports):
     #not deleted, last version
-    #deleted_adult_reports = get_deleted_adult_reports(reports)
-    # not deleted
-    undeleted = reports.exclude(version_UUID__in=RawSQL("select \"version_UUID\" from tigaserver_app_report r, (select report_id, user_id, count(\"version_UUID\") from tigaserver_app_report where type = 'adult' and report_id in (select distinct report_id from tigaserver_app_report where version_number = -1) group by report_id, user_id having count(\"version_UUID\") >1) as deleted where r.report_id = deleted.report_id and r.user_id = deleted.user_id",()))
-    # last version
-    latest_versions = undeleted.filter(version_UUID__in=RawSQL("select \"version_UUID\" from tigaserver_app_report r,(select report_id,max(version_number) as higher from tigaserver_app_report where type = 'adult' group by report_id) maxes where r.type = 'adult' and r.report_id = maxes.report_id and r.version_number = maxes.higher",()))
-    #latest_versions = undeleted.filter(version_UUID__in=RawSQL("select \"version_UUID\" from tigaserver_app_report r,(select report_id,max(version_number) as higher from tigaserver_app_report where type = 'adult' group by report_id) maxes where r.type = 'adult' and r.report_id = maxes.report_id and r.version_number = maxes.higher union select \"version_UUID\" from tigaserver_app_report r, (select report_id, user_id, version_number, count(\"version_UUID\") from tigaserver_app_report where type = 'adult' group by report_id, user_id, version_number having count(\"version_UUID\") = 1) as uniq where r.report_id = uniq.report_id and r.user_id = uniq.user_id and r.version_number = uniq.version_number",()))
-    return latest_versions
+    return reports.filter(type=Report.TYPE_ADULT).non_deleted()
 
 
 def assign_reports_to_national_supervisor(this_user):

tigacrafting/templates/tigacrafting/expert_report_status.html

@@ -240,13 +240,6 @@ <h5>
                     <strong>WARNING: THIS REPORT HAS BEEN DELETED BY THE USER. IT WILL NOT BE SENT FOR EXPERT VALIDATION OR DISPLAYED ON THE PUBLIC MAP.</strong></div>
             </div>
 {% endif %}
-{% if not report.deleted and not report.latest_version %}
-                    <div class="row">
-                <div class="col-md-12">
-                    <strong>WARNING: THIS IS NOT THE LATEST VERSION OF THIS REPORT. IT WILL NOT BE SENT FOR EXPERT VALIDATION OR DISPLAYED ON THE PUBLIC MAP.</strong><br>The latest version of this report is <a href="{% url 'expert_report_status' %}?version_uuid={{ report.get_which_is_latest }}">{{ report.get_which_is_latest }}</a></div>
-            </div>
-{% endif %}
-
             <div class="row" style="margin-bottom: 3px;">
         <div class="col-md-12">
         {% if report.get_is_expert_validated %}

tigacrafting/templates/tigacrafting/single_report_view.html

@@ -187,13 +187,6 @@ <h5>
                     </div>
                 </div>
             {% endif %}
-            {% if not report.deleted and not report.latest_version %}
-                <div class="row">
-                    <div class="col-md-12">
-                        <strong>WARNING: THIS IS NOT THE LATEST VERSION OF THIS REPORT. IT WILL NOT BE SENT FOR EXPERT VALIDATION OR DISPLAYED ON THE PUBLIC MAP.</strong><br>The latest version of this report is <a href="{% url 'expert_report_status' %}?version_uuid={{ report.get_which_is_latest }}">{{ report.get_which_is_latest }}</a>
-                    </div>
-                </div>
-            {% endif %}
             <div class="row" style="margin-bottom: 3px;">
                 <div class="col-md-12">
                     {% if report.get_is_expert_validated %}

tigacrafting/views.py

@@ -44,7 +44,6 @@
 from django.template.loader import TemplateDoesNotExist
 from django.utils.translation import gettext as _
 from tigacrafting.querystring_parser import parser
-from django.db.models.expressions import RawSQL
 import functools
 import operator
 import math
@@ -341,7 +340,7 @@ def show_processing(request):
 
 
 def filter_tasks(tasks):
-    tasks_filtered = filter(lambda x: not x.photo.report.deleted and x.photo.report.latest_version, tasks)
+    tasks_filtered = filter(lambda x: not x.photo.report.deleted, tasks)
     return tasks_filtered
 
 def fast_filter_reports(reports):
@@ -356,11 +355,12 @@ def fast_filter_reports(reports):
 
 
 def filter_reports(reports, sort=True):
+    reports_filtered = reports.non_deleted()
+
     if sort:
-        reports_filtered = sorted(filter(lambda x: not x.deleted and x.latest_version, reports), key=attrgetter('n_annotations'), reverse=True)
-    else:
-        reports_filtered = filter(lambda x: not x.deleted and x.latest_version, reports)
-    return reports_filtered
+        reports_filtered = sorted(reports_filtered, key=attrgetter('n_annotations'), reverse=True)
+
+    return list(reports_filtered)
 
 
 def filter_spain_reports(reports, sort=True):
@@ -380,7 +380,8 @@ def filter_eu_reports(reports, sort=True):
 
 
 def filter_reports_for_superexpert(reports):
-    reports_filtered = filter(lambda x: not x.deleted and x.latest_version and len(list(filter(lambda y: y.is_expert() and y.validation_complete, x.expert_report_annotations.all()))) >= 3, reports)
+    reports_filtered = reports.non_deleted()
+    reports_filtered = filter(lambda x: len(list(filter(lambda y: y.is_expert() and y.validation_complete, x.expert_report_annotations.all()))) >= 3, reports_filtered)
     return reports_filtered
 
 
@@ -1685,17 +1686,10 @@ def picture_validation(request,tasks_per_page='300',visibility='visible', usr_no
         #
         # new_reports_unfiltered = list(new_reports_unfiltered)
 
-        #report_id_deleted_reports = Report.objects.filter(version_UUID__in=RawSQL("select \"version_UUID\" from tigaserver_app_report r, (select report_id, user_id, count(\"version_UUID\") from tigaserver_app_report where type = 'adult' and report_id in (select distinct report_id from tigaserver_app_report where version_number = -1) group by report_id, user_id having count(\"version_UUID\") >1) as deleted where r.report_id = deleted.report_id and r.user_id = deleted.user_id",())).values("report_id").distinct()
-        report_id_deleted_reports_adults = Report.objects.filter(version_UUID__in=RawSQL("select \"version_UUID\" from tigaserver_app_report r, (select report_id, user_id, count(\"version_UUID\") from tigaserver_app_report where type = 'adult' and report_id in (select distinct report_id from tigaserver_app_report where version_number = -1) group by report_id, user_id having count(\"version_UUID\") >1) as deleted where r.report_id = deleted.report_id and r.user_id = deleted.user_id",())).values("version_UUID").distinct()
-        report_id_deleted_reports_sites = Report.objects.filter(version_UUID__in=RawSQL("select \"version_UUID\" from tigaserver_app_report r, (select report_id, user_id, count(\"version_UUID\") from tigaserver_app_report where type = 'site' and report_id in (select distinct report_id from tigaserver_app_report where version_number = -1) group by report_id, user_id having count(\"version_UUID\") >1) as deleted where r.report_id = deleted.report_id and r.user_id = deleted.user_id",())).values("version_UUID").distinct()
-
-        new_reports_unfiltered = new_reports_unfiltered.exclude(version_UUID__in=report_id_deleted_reports_adults).exclude(version_UUID__in=report_id_deleted_reports_sites)
-
-        new_reports_unfiltered = new_reports_unfiltered.filter(version_UUID__in=RawSQL("select \"version_UUID\" from tigaserver_app_report r,(select report_id,max(version_number) as higher from tigaserver_app_report where type = 'adult' group by report_id) maxes where r.type = 'adult' and r.report_id = maxes.report_id and r.version_number = maxes.higher union select \"version_UUID\" from tigaserver_app_report r, (select report_id,max(version_number) as higher from tigaserver_app_report where type = 'site' group by report_id) maxes where r.type = 'site' and r.report_id = maxes.report_id and r.version_number = maxes.higher",()))
-        #new_reports_unfiltered = new_reports_unfiltered.filter(version_UUID__in=RawSQL("select \"version_UUID\" from tigaserver_app_report r,(select report_id,max(version_number) as higher from tigaserver_app_report where type = 'adult' group by report_id) maxes where r.type = 'adult' and r.report_id = maxes.report_id and r.version_number = maxes.higher union select \"version_UUID\" from tigaserver_app_report r, (select report_id, user_id, version_number, count(\"version_UUID\") from tigaserver_app_report where type = 'adult' group by report_id, user_id, version_number having count(\"version_UUID\") = 1) as uniq where r.report_id = uniq.report_id and r.user_id = uniq.user_id and r.version_number = uniq.version_number union select \"version_UUID\" from tigaserver_app_report r,(select report_id,max(version_number) as higher from tigaserver_app_report where type = 'site' group by report_id) maxes where r.type = 'site' and r.report_id = maxes.report_id and r.version_number = maxes.higher union select \"version_UUID\" from tigaserver_app_report r, (select report_id, user_id, version_number, count(\"version_UUID\") from tigaserver_app_report where type = 'site' group by report_id, user_id, version_number having count(\"version_UUID\") = 1) as uniq where r.report_id = uniq.report_id and r.user_id = uniq.user_id and r.version_number = uniq.version_number",()))
+        new_reports_unfiltered = new_reports_unfiltered.non_deleted()
 
         # for r in new_reports_unfiltered:
-        #      print("{0} {1} {2}".format(r.version_UUID, r.deleted, r.latest_version))
+        #      print("{0} {1}".format(r.version_UUID, r.deleted))
 
         paginator = Paginator(new_reports_unfiltered, int(tasks_per_page))
         page = request.GET.get('page', 1)

tigamap/templates/tigamap/validated_report_map_filterable.html

@@ -622,8 +622,8 @@
 
 function add_to_layer(data, this_layer, filter) {
     $.each(data, function (index, value) {
-        //if (value != null && value.latest_version && value.visible && value.lat != null && value.lon != null && value.type != null && value.type == selected_type && (!filter || ( (selected_year == 0 || value.creation_year == selected_year) && (selected_month == 0 || value.creation_month == selected_month) && (selected_validation == -2 || (value.movelab_annotation != null && ((value.type == 'adult' && value.movelab_annotation.tiger_certainty_category != null && value.movelab_annotation.tiger_certainty_category >= selected_validation) || (value.type == 'site' && value.movelab_annotation.site_certainty_category != null && value.movelab_annotation.site_certainty_category >= selected_validation) ) ))))) {
-        if (value != null && value.latest_version && value.visible && value.lat != null && value.lon != null && value.type != null && value.type == selected_type && (!filter || ( (selected_year == 0 || value.creation_year == selected_year) && (selected_month == 0 || value.creation_month == selected_month) && (selected_validation == -2 || (value.movelab_annotation != null && ((value.type == 'adult' && value.movelab_annotation.tiger_certainty_category != null && value.movelab_annotation.tiger_certainty_category >= selected_validation) || (value.type == 'adult' && value.movelab_annotation.aegypti_certainty_category != null && value.movelab_annotation.aegypti_certainty_category >= selected_validation) || (value.type == 'site' && value.movelab_annotation.site_certainty_category != null && value.movelab_annotation.site_certainty_category >= selected_validation) ) ))))) {
+        //if (value != null && value.visible && value.lat != null && value.lon != null && value.type != null && value.type == selected_type && (!filter || ( (selected_year == 0 || value.creation_year == selected_year) && (selected_month == 0 || value.creation_month == selected_month) && (selected_validation == -2 || (value.movelab_annotation != null && ((value.type == 'adult' && value.movelab_annotation.tiger_certainty_category != null && value.movelab_annotation.tiger_certainty_category >= selected_validation) || (value.type == 'site' && value.movelab_annotation.site_certainty_category != null && value.movelab_annotation.site_certainty_category >= selected_validation) ) ))))) {
+        if (value != null && value.visible && value.lat != null && value.lon != null && value.type != null && value.type == selected_type && (!filter || ( (selected_year == 0 || value.creation_year == selected_year) && (selected_month == 0 || value.creation_month == selected_month) && (selected_validation == -2 || (value.movelab_annotation != null && ((value.type == 'adult' && value.movelab_annotation.tiger_certainty_category != null && value.movelab_annotation.tiger_certainty_category >= selected_validation) || (value.type == 'adult' && value.movelab_annotation.aegypti_certainty_category != null && value.movelab_annotation.aegypti_certainty_category >= selected_validation) || (value.type == 'site' && value.movelab_annotation.site_certainty_category != null && value.movelab_annotation.site_certainty_category >= selected_validation) ) ))))) {
             this_layer.addLayer(L.marker([value.lat, value.lon], {icon: get_icon(value)}).bindPopup(make_popup_text(value), popup_options));
         }
     });