Skip to content

Commit e63dcbf

Browse files
epouepou
committed
Report versioning
1 parent 2c83ba9 commit e63dcbf

26 files changed

+737
-278
lines changed

‎requirements/prod_20_04.pip‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ django-import-export==1.2.0
2727
django-leaflet==0.13.2
2828
django-messages==0.6.0
2929
django-rest-swagger==2.2.0
30+
django-simple-history==3.0.0
3031
django-sslserver==0.22
3132
django-taggit==1.1.0
3233
djangorestframework==3.10.3

‎stats/views.py‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1002,7 +1002,7 @@ def show_fix_users(request):
10021002

10031003

10041004
def show_report_users(request):
1005-
real_reports = [report for report in Report.objects.filter(Q(package_name='Tigatrapp', creation_time__gte=date(2014, 6, 24)) | Q(package_name='ceab.movelab.tigatrapp', package_version__gt=3)) if report.latest_version]
1005+
real_reports = [report for report in Report.objects.filter(Q(package_name='Tigatrapp', creation_time__gte=date(2014, 6, 24)) | Q(package_name='ceab.movelab.tigatrapp', package_version__gt=3))]
10061006
tz = get_localzone()
10071007
ref_date = datetime.datetime(2014, 6, 13, tzinfo=tz)
10081008
end_date = tz.localize(datetime.datetime.now())
@@ -1073,10 +1073,9 @@ def get_hashtag_map_data(request):
10731073
r = Report.objects.filter(note__icontains=hashtag).order_by('-server_upload_time')[:200]
10741074
n = 0
10751075
for report in r:
1076-
if report.latest_version:
1077-
n = n + 1
1078-
dates.append(report.server_upload_time)
1079-
data.append({ 'note': report.note, 'picture': report.photo_html, 'lat': report.lat, 'lon': report.lon, 'date_uploaded': report.server_upload_time.strftime('%d-%m-%Y / %H:%M:%S') })
1076+
n = n + 1
1077+
dates.append(report.server_upload_time)
1078+
data.append({ 'note': report.note, 'picture': report.photo_html, 'lat': report.lat, 'lon': report.lon, 'date_uploaded': report.server_upload_time.strftime('%d-%m-%Y / %H:%M:%S') })
10801079
min_date_str = ''
10811080
max_date_str = ''
10821081
if len(dates) == 0:

‎tigacrafting/criteria.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,12 +12,12 @@ def filter_users_by_score(score):
1212
return score_user_ids
1313

1414
def filter_users_with_storm_drain_pictures(reports):
15-
reports_filtered = filter(lambda x: not x.deleted and x.latest_version and x.embornals, reports)
15+
reports_filtered = reports.non_deleted()
16+
reports_filtered = filter(lambda x: x.embornals, reports_filtered)
1617
return reports_filtered
1718

1819
def filter_users_with_pictures(reports):
19-
reports_filtered = filter(lambda x: not x.deleted and x.latest_version, reports)
20-
return reports_filtered
20+
return list(reports.non_deleted().all())
2121

2222
def users_with_pictures():
2323
return filter_reports('users_with_pictures')

‎tigacrafting/migrations/0020_auto_20240125_1557.py‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
# Generated by Django 2.2.7 on 2024-01-25 15:57
2+
3+
from django.db import migrations
4+
5+
6+
class Migration(migrations.Migration):
7+
8+
dependencies = [
9+
('tigacrafting', '0019_auto_20240122_1709'),
10+
]
11+
12+
operations = [
13+
migrations.RemoveConstraint(
14+
model_name='expertreportannotation',
15+
name='unique_assignation',
16+
)
17+
]

‎tigacrafting/migrations/0021_auto_20240125_1557.py‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
# Generated by Django 2.2.7 on 2024-01-25 15:57
2+
3+
from django.db import migrations, models
4+
from django.db.models import OuterRef, Subquery
5+
6+
def remove_expert_report_annotation_duplicates(apps, schema_editor):
7+
ExpertReportAnnotation = apps.get_model("tigacrafting", "ExpertReportAnnotation")
8+
9+
# Will keep only the one with the most recent last_modified date.
10+
last_annotation_subquery = ExpertReportAnnotation.objects.filter(
11+
user=OuterRef('user'),
12+
report=OuterRef('report')
13+
).order_by('-last_modified').values('pk')[:1]
14+
15+
print("Deleting duplicate ExpertReportAnnotation, keeping last modified...")
16+
print(ExpertReportAnnotation.objects.exclude(pk__in=Subquery(last_annotation_subquery)).delete())
17+
18+
19+
class Migration(migrations.Migration):
20+
21+
dependencies = [
22+
('tigacrafting', '0020_auto_20240125_1557'),
23+
('tigaserver_app', '0050_report_versioning'),
24+
]
25+
26+
operations = [
27+
migrations.RunPython(remove_expert_report_annotation_duplicates, migrations.RunPython.noop),
28+
migrations.AddConstraint(
29+
model_name='expertreportannotation',
30+
constraint=models.UniqueConstraint(fields=('user', 'report'), name='unique_assignation'),
31+
),
32+
]

‎tigacrafting/report_queues.py‎

Lines changed: 13 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
from tigaserver_app.models import Report, ExpertReportAnnotation, EuropeCountry
22
from django.core.exceptions import ObjectDoesNotExist
33
from tigacrafting.models import UserStat
4-
from django.db.models.expressions import RawSQL
54
from django.contrib.auth.models import User
65
from django.db.models import Count
76
from django.db.models import Q
@@ -56,28 +55,28 @@ def filter_reports_for_superexpert(reports):
5655
# not deleted, last version, completely validated by at least three experts
5756
#deleted_adult_reports = get_deleted_adult_reports(reports)
5857
# not deleted
59-
undeleted = reports.exclude(version_UUID__in=RawSQL("select \"version_UUID\" from tigaserver_app_report r, (select report_id, user_id, count(\"version_UUID\") from tigaserver_app_report where type = 'adult' and report_id in (select distinct report_id from tigaserver_app_report where version_number = -1) group by report_id, user_id having count(\"version_UUID\") >1) as deleted where r.report_id = deleted.report_id and r.user_id = deleted.user_id",()))
60-
# last version
61-
latest_versions = undeleted.filter(version_UUID__in=RawSQL("select \"version_UUID\" from tigaserver_app_report r,(select report_id,max(version_number) as higher from tigaserver_app_report where type = 'adult' group by report_id) maxes where r.type = 'adult' and r.report_id = maxes.report_id and r.version_number = maxes.higher",()))
62-
#latest_versions = undeleted.filter(version_UUID__in=RawSQL("select \"version_UUID\" from tigaserver_app_report r,(select report_id,max(version_number) as higher from tigaserver_app_report where type = 'adult' group by report_id) maxes where r.type = 'adult' and r.report_id = maxes.report_id and r.version_number = maxes.higher union select \"version_UUID\" from tigaserver_app_report r, (select report_id, user_id, version_number, count(\"version_UUID\") from tigaserver_app_report where type = 'adult' group by report_id, user_id, version_number having count(\"version_UUID\") = 1) as uniq where r.report_id = uniq.report_id and r.user_id = uniq.user_id and r.version_number = uniq.version_number",()))
58+
reports_qs = reports.filter(type=Report.TYPE_ADULT).non_deleted()
6359
# fully validated
6460
experts = User.objects.filter(groups__name='expert')
65-
fully_validated = ExpertReportAnnotation.objects.filter(report__in=latest_versions).filter(user__in=experts).filter(validation_complete=True).values('report').annotate(n_validations=Count('report')).filter(n_validations__gte=MAX_N_OF_EXPERTS_ASSIGNED_PER_REPORT)
61+
fully_validated = ExpertReportAnnotation.objects.filter(
62+
report__in=reports_qs,
63+
user__in=experts,
64+
validation_complete=True
65+
).values('report').annotate(
66+
n_validations=Count('report')
67+
).filter(
68+
n_validations__gte=MAX_N_OF_EXPERTS_ASSIGNED_PER_REPORT
69+
)
70+
6671
report_ids = set(fully_validated.values_list('report', flat=True))
67-
filtered_reports = latest_versions.filter(version_UUID__in=report_ids)
72+
filtered_reports = reports_qs.filter(version_UUID__in=report_ids)
6873
#filtered_reports = filter(lambda x: len(list(filter(lambda y: y.is_expert() and y.validation_complete, x.expert_report_annotations.all()))) >= 3, latest_versions)
6974
return filtered_reports
7075

7176

7277
def filter_reports(reports):
7378
#not deleted, last version
74-
#deleted_adult_reports = get_deleted_adult_reports(reports)
75-
# not deleted
76-
undeleted = reports.exclude(version_UUID__in=RawSQL("select \"version_UUID\" from tigaserver_app_report r, (select report_id, user_id, count(\"version_UUID\") from tigaserver_app_report where type = 'adult' and report_id in (select distinct report_id from tigaserver_app_report where version_number = -1) group by report_id, user_id having count(\"version_UUID\") >1) as deleted where r.report_id = deleted.report_id and r.user_id = deleted.user_id",()))
77-
# last version
78-
latest_versions = undeleted.filter(version_UUID__in=RawSQL("select \"version_UUID\" from tigaserver_app_report r,(select report_id,max(version_number) as higher from tigaserver_app_report where type = 'adult' group by report_id) maxes where r.type = 'adult' and r.report_id = maxes.report_id and r.version_number = maxes.higher",()))
79-
#latest_versions = undeleted.filter(version_UUID__in=RawSQL("select \"version_UUID\" from tigaserver_app_report r,(select report_id,max(version_number) as higher from tigaserver_app_report where type = 'adult' group by report_id) maxes where r.type = 'adult' and r.report_id = maxes.report_id and r.version_number = maxes.higher union select \"version_UUID\" from tigaserver_app_report r, (select report_id, user_id, version_number, count(\"version_UUID\") from tigaserver_app_report where type = 'adult' group by report_id, user_id, version_number having count(\"version_UUID\") = 1) as uniq where r.report_id = uniq.report_id and r.user_id = uniq.user_id and r.version_number = uniq.version_number",()))
80-
return latest_versions
79+
return reports.filter(type=Report.TYPE_ADULT).non_deleted()
8180

8281

8382
def assign_reports_to_national_supervisor(this_user):

‎tigacrafting/templates/tigacrafting/expert_report_status.html‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -240,13 +240,6 @@ <h5>
240240
<strong>WARNING: THIS REPORT HAS BEEN DELETED BY THE USER. IT WILL NOT BE SENT FOR EXPERT VALIDATION OR DISPLAYED ON THE PUBLIC MAP.</strong></div>
241241
</div>
242242
{% endif %}
243-
{% if not report.deleted and not report.latest_version %}
244-
<div class="row">
245-
<div class="col-md-12">
246-
<strong>WARNING: THIS IS NOT THE LATEST VERSION OF THIS REPORT. IT WILL NOT BE SENT FOR EXPERT VALIDATION OR DISPLAYED ON THE PUBLIC MAP.</strong><br>The latest version of this report is <a href="{% url 'expert_report_status' %}?version_uuid={{ report.get_which_is_latest }}">{{ report.get_which_is_latest }}</a></div>
247-
</div>
248-
{% endif %}
249-
250243
<div class="row" style="margin-bottom: 3px;">
251244
<div class="col-md-12">
252245
{% if report.get_is_expert_validated %}

‎tigacrafting/templates/tigacrafting/single_report_view.html‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -187,13 +187,6 @@ <h5>
187187
</div>
188188
</div>
189189
{% endif %}
190-
{% if not report.deleted and not report.latest_version %}
191-
<div class="row">
192-
<div class="col-md-12">
193-
<strong>WARNING: THIS IS NOT THE LATEST VERSION OF THIS REPORT. IT WILL NOT BE SENT FOR EXPERT VALIDATION OR DISPLAYED ON THE PUBLIC MAP.</strong><br>The latest version of this report is <a href="{% url 'expert_report_status' %}?version_uuid={{ report.get_which_is_latest }}">{{ report.get_which_is_latest }}</a>
194-
</div>
195-
</div>
196-
{% endif %}
197190
<div class="row" style="margin-bottom: 3px;">
198191
<div class="col-md-12">
199192
{% if report.get_is_expert_validated %}

‎tigacrafting/views.py‎

Lines changed: 10 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,6 @@
4444
from django.template.loader import TemplateDoesNotExist
4545
from django.utils.translation import gettext as _
4646
from tigacrafting.querystring_parser import parser
47-
from django.db.models.expressions import RawSQL
4847
import functools
4948
import operator
5049
import math
@@ -341,7 +340,7 @@ def show_processing(request):
341340

342341

343342
def filter_tasks(tasks):
344-
tasks_filtered = filter(lambda x: not x.photo.report.deleted and x.photo.report.latest_version, tasks)
343+
tasks_filtered = filter(lambda x: not x.photo.report.deleted, tasks)
345344
return tasks_filtered
346345

347346
def fast_filter_reports(reports):
@@ -356,11 +355,12 @@ def fast_filter_reports(reports):
356355

357356

358357
def filter_reports(reports, sort=True):
358+
reports_filtered = reports.non_deleted()
359+
359360
if sort:
360-
reports_filtered = sorted(filter(lambda x: not x.deleted and x.latest_version, reports), key=attrgetter('n_annotations'), reverse=True)
361-
else:
362-
reports_filtered = filter(lambda x: not x.deleted and x.latest_version, reports)
363-
return reports_filtered
361+
reports_filtered = sorted(reports_filtered, key=attrgetter('n_annotations'), reverse=True)
362+
363+
return list(reports_filtered)
364364

365365

366366
def filter_spain_reports(reports, sort=True):
@@ -380,7 +380,8 @@ def filter_eu_reports(reports, sort=True):
380380

381381

382382
def filter_reports_for_superexpert(reports):
383-
reports_filtered = filter(lambda x: not x.deleted and x.latest_version and len(list(filter(lambda y: y.is_expert() and y.validation_complete, x.expert_report_annotations.all()))) >= 3, reports)
383+
reports_filtered = reports.non_deleted()
384+
reports_filtered = filter(lambda x: len(list(filter(lambda y: y.is_expert() and y.validation_complete, x.expert_report_annotations.all()))) >= 3, reports_filtered)
384385
return reports_filtered
385386

386387

@@ -1685,17 +1686,10 @@ def picture_validation(request,tasks_per_page='300',visibility='visible', usr_no
16851686
#
16861687
# new_reports_unfiltered = list(new_reports_unfiltered)
16871688

1688-
#report_id_deleted_reports = Report.objects.filter(version_UUID__in=RawSQL("select \"version_UUID\" from tigaserver_app_report r, (select report_id, user_id, count(\"version_UUID\") from tigaserver_app_report where type = 'adult' and report_id in (select distinct report_id from tigaserver_app_report where version_number = -1) group by report_id, user_id having count(\"version_UUID\") >1) as deleted where r.report_id = deleted.report_id and r.user_id = deleted.user_id",())).values("report_id").distinct()
1689-
report_id_deleted_reports_adults = Report.objects.filter(version_UUID__in=RawSQL("select \"version_UUID\" from tigaserver_app_report r, (select report_id, user_id, count(\"version_UUID\") from tigaserver_app_report where type = 'adult' and report_id in (select distinct report_id from tigaserver_app_report where version_number = -1) group by report_id, user_id having count(\"version_UUID\") >1) as deleted where r.report_id = deleted.report_id and r.user_id = deleted.user_id",())).values("version_UUID").distinct()
1690-
report_id_deleted_reports_sites = Report.objects.filter(version_UUID__in=RawSQL("select \"version_UUID\" from tigaserver_app_report r, (select report_id, user_id, count(\"version_UUID\") from tigaserver_app_report where type = 'site' and report_id in (select distinct report_id from tigaserver_app_report where version_number = -1) group by report_id, user_id having count(\"version_UUID\") >1) as deleted where r.report_id = deleted.report_id and r.user_id = deleted.user_id",())).values("version_UUID").distinct()
1691-
1692-
new_reports_unfiltered = new_reports_unfiltered.exclude(version_UUID__in=report_id_deleted_reports_adults).exclude(version_UUID__in=report_id_deleted_reports_sites)
1693-
1694-
new_reports_unfiltered = new_reports_unfiltered.filter(version_UUID__in=RawSQL("select \"version_UUID\" from tigaserver_app_report r,(select report_id,max(version_number) as higher from tigaserver_app_report where type = 'adult' group by report_id) maxes where r.type = 'adult' and r.report_id = maxes.report_id and r.version_number = maxes.higher union select \"version_UUID\" from tigaserver_app_report r, (select report_id,max(version_number) as higher from tigaserver_app_report where type = 'site' group by report_id) maxes where r.type = 'site' and r.report_id = maxes.report_id and r.version_number = maxes.higher",()))
1695-
#new_reports_unfiltered = new_reports_unfiltered.filter(version_UUID__in=RawSQL("select \"version_UUID\" from tigaserver_app_report r,(select report_id,max(version_number) as higher from tigaserver_app_report where type = 'adult' group by report_id) maxes where r.type = 'adult' and r.report_id = maxes.report_id and r.version_number = maxes.higher union select \"version_UUID\" from tigaserver_app_report r, (select report_id, user_id, version_number, count(\"version_UUID\") from tigaserver_app_report where type = 'adult' group by report_id, user_id, version_number having count(\"version_UUID\") = 1) as uniq where r.report_id = uniq.report_id and r.user_id = uniq.user_id and r.version_number = uniq.version_number union select \"version_UUID\" from tigaserver_app_report r,(select report_id,max(version_number) as higher from tigaserver_app_report where type = 'site' group by report_id) maxes where r.type = 'site' and r.report_id = maxes.report_id and r.version_number = maxes.higher union select \"version_UUID\" from tigaserver_app_report r, (select report_id, user_id, version_number, count(\"version_UUID\") from tigaserver_app_report where type = 'site' group by report_id, user_id, version_number having count(\"version_UUID\") = 1) as uniq where r.report_id = uniq.report_id and r.user_id = uniq.user_id and r.version_number = uniq.version_number",()))
1689+
new_reports_unfiltered = new_reports_unfiltered.non_deleted()
16961690

16971691
# for r in new_reports_unfiltered:
1698-
# print("{0} {1} {2}".format(r.version_UUID, r.deleted, r.latest_version))
1692+
# print("{0} {1}".format(r.version_UUID, r.deleted))
16991693

17001694
paginator = Paginator(new_reports_unfiltered, int(tasks_per_page))
17011695
page = request.GET.get('page', 1)

‎tigamap/templates/tigamap/validated_report_map_filterable.html‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -622,8 +622,8 @@
622622

623623
function add_to_layer(data, this_layer, filter) {
624624
$.each(data, function (index, value) {
625-
//if (value != null && value.latest_version && value.visible && value.lat != null && value.lon != null && value.type != null && value.type == selected_type && (!filter || ( (selected_year == 0 || value.creation_year == selected_year) && (selected_month == 0 || value.creation_month == selected_month) && (selected_validation == -2 || (value.movelab_annotation != null && ((value.type == 'adult' && value.movelab_annotation.tiger_certainty_category != null && value.movelab_annotation.tiger_certainty_category >= selected_validation) || (value.type == 'site' && value.movelab_annotation.site_certainty_category != null && value.movelab_annotation.site_certainty_category >= selected_validation) ) ))))) {
626-
if (value != null && value.latest_version && value.visible && value.lat != null && value.lon != null && value.type != null && value.type == selected_type && (!filter || ( (selected_year == 0 || value.creation_year == selected_year) && (selected_month == 0 || value.creation_month == selected_month) && (selected_validation == -2 || (value.movelab_annotation != null && ((value.type == 'adult' && value.movelab_annotation.tiger_certainty_category != null && value.movelab_annotation.tiger_certainty_category >= selected_validation) || (value.type == 'adult' && value.movelab_annotation.aegypti_certainty_category != null && value.movelab_annotation.aegypti_certainty_category >= selected_validation) || (value.type == 'site' && value.movelab_annotation.site_certainty_category != null && value.movelab_annotation.site_certainty_category >= selected_validation) ) ))))) {
625+
//if (value != null && value.visible && value.lat != null && value.lon != null && value.type != null && value.type == selected_type && (!filter || ( (selected_year == 0 || value.creation_year == selected_year) && (selected_month == 0 || value.creation_month == selected_month) && (selected_validation == -2 || (value.movelab_annotation != null && ((value.type == 'adult' && value.movelab_annotation.tiger_certainty_category != null && value.movelab_annotation.tiger_certainty_category >= selected_validation) || (value.type == 'site' && value.movelab_annotation.site_certainty_category != null && value.movelab_annotation.site_certainty_category >= selected_validation) ) ))))) {
626+
if (value != null && value.visible && value.lat != null && value.lon != null && value.type != null && value.type == selected_type && (!filter || ( (selected_year == 0 || value.creation_year == selected_year) && (selected_month == 0 || value.creation_month == selected_month) && (selected_validation == -2 || (value.movelab_annotation != null && ((value.type == 'adult' && value.movelab_annotation.tiger_certainty_category != null && value.movelab_annotation.tiger_certainty_category >= selected_validation) || (value.type == 'adult' && value.movelab_annotation.aegypti_certainty_category != null && value.movelab_annotation.aegypti_certainty_category >= selected_validation) || (value.type == 'site' && value.movelab_annotation.site_certainty_category != null && value.movelab_annotation.site_certainty_category >= selected_validation) ) ))))) {
627627
this_layer.addLayer(L.marker([value.lat, value.lon], {icon: get_icon(value)}).bindPopup(make_popup_text(value), popup_options));
628628
}
629629
});

0 commit comments

Comments
 (0)