add admin UI to import schools

Author: gardenerik

bullet/bullet_admin/forms/education.py

@@ -1,9 +1,144 @@
+import csv
+from dataclasses import dataclass
+from functools import partial
+from io import TextIOWrapper
+
 from django import forms
-from education.models import School
+from django.core.validators import FileExtensionValidator
+from django.db import transaction
+from django_countries.fields import CountryField
+from education.models import School, SchoolType
+from education.tasks import send_schools_to_search
 
 from bullet_admin.forms.utils import get_country_choices
 
 
+class SchoolCSVImportForm(forms.Form):
+    csv_file = forms.FileField(
+        label="CSV file",
+        validators=[FileExtensionValidator(["csv"])],
+        help_text="CSV format: identifier, name, address, types (comma-separated)",
+    )
+    country = CountryField().formfield()
+    preview = forms.BooleanField(
+        label="Preview only (don't import)",
+        required=False,
+        initial=True,
+        help_text="Check this to preview the import without making changes",
+    )
+
+    @dataclass
+    class SchoolData:
+        identifier: str
+        name: str
+        address: str
+        types: list[str]
+
+    def __init__(self, competition, user, **kwargs):
+        super().__init__(**kwargs)
+        self.fields["country"].choices = get_country_choices(competition, user)
+        self.competition = competition
+        self.user = user
+
+    def parse_csv(self) -> list[SchoolData]:
+        csv_file = self.cleaned_data["csv_file"]
+
+        schools = []
+        with TextIOWrapper(csv_file.file, encoding="utf-8") as decoded_file:
+            csv_file.file.seek(0)
+
+            reader = csv.DictReader(decoded_file)
+            if not reader.fieldnames:
+                raise ValueError("CSV file has no header row")
+
+            required_columns = ["identifier", "name", "address", "types"]
+            missing_columns = []
+            for col in required_columns:
+                if col not in reader.fieldnames:
+                    missing_columns.append(col)
+            if missing_columns:
+                raise ValueError(
+                    f"CSV file is missing required columns: {', '.join(missing_columns)}."
+                )
+
+            for row in reader:
+                identifier = row.get("identifier", "").strip()
+                name = row.get("name", "").strip()
+                address = row.get("address", "").strip()
+                types_str = row.get("types", "").strip()
+                types = [t.strip() for t in types_str.split(",") if t.strip()]
+
+                schools.append(
+                    SchoolCSVImportForm.SchoolData(identifier, name, address, types)
+                )
+
+        if not schools:
+            raise ValueError("No valid school data found in CSV file")
+        return schools
+
+    @transaction.atomic
+    def import_schools(self):
+        schools_data = self.parse_csv()
+        country = self.cleaned_data["country"]
+
+        created_count = 0
+        updated_count = 0
+        errors = []
+
+        school_types_cache = {}
+        used_types = set()
+        for data in schools_data:
+            used_types.update(data.types)
+
+        for type_name in used_types:
+            school_type = SchoolType.objects.filter(identifier=type_name).first()
+            if not school_type:
+                continue
+            school_types_cache[type_name] = school_type
+
+        school_ids = []
+        for data in schools_data:
+            school = None
+            if data.identifier:
+                school = School.objects.filter(
+                    importer_identifier=data.identifier,
+                    country=country,
+                ).first()
+
+            if school:
+                if school.importer_ignored:
+                    errors.append(f"School {school.importer_identifier} is ignored")
+                    continue
+                updated_count += 1
+            else:
+                school = School(
+                    importer_identifier=data.identifier,
+                    country=country,
+                )
+                created_count += 1
+
+            school.name = data.name
+            school.address = data.address
+            school.save(send_to_search=False)
+
+            types = []
+            for type_name in data.types:
+                if type_name in school_types_cache:
+                    types.append(school_types_cache[type_name])
+            school.types.set(types)
+
+            school_ids.append(school.id)
+
+        transaction.on_commit(partial(send_schools_to_search, school_ids=school_ids))
+
+        return {
+            "created": created_count,
+            "updated": updated_count,
+            "errors": errors,
+            "total": len(schools_data),
+        }
+
+
 class SchoolForm(forms.ModelForm):
     class Meta:
         model = School

bullet/bullet_admin/templates/bullet_admin/education/school_csv_import.html

@@ -0,0 +1,43 @@
+{% extends "bullet_admin/generic/form.html" %}
+{% load badmin %}
+
+{% block after_form %}
+  {% if preview_data is not None %}
+  <div class="mt-8">
+    <h3 class="text-lg font-semibold mb-4">Preview</h3>
+    <div class="bg-blue-50 border border-blue-200 rounded-md p-4 mb-4">
+        <p class="text-sm text-blue-800">
+          This is a preview. No schools have been imported yet.
+          Uncheck "Preview only" and submit again to perform the actual import.
+        </p>
+    </div>
+
+    <div class="overflow-x-auto">
+      <table class="min-w-full divide-y divide-gray-200">
+        <thead class="bg-gray-50">
+          <tr>
+            <th class="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider">Identifier</th>
+            <th class="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider">Name</th>
+            <th class="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider">Address</th>
+            <th class="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider">Types</th>
+          </tr>
+        </thead>
+        <tbody class="bg-white divide-y divide-gray-200">
+            {% for school in preview_data %}
+                <tr class="{% cycle "bg-white" "bg-gray-50" %}">
+                  <td class="px-6 py-4 whitespace-nowrap text-sm text-gray-900">{{ school.identifier }}</td>
+                  <td class="px-6 py-4 text-sm text-gray-900">{{ school.name }}</td>
+                  <td class="px-6 py-4 text-sm text-gray-900">{{ school.address }}</td>
+                  <td class="px-6 py-4 text-sm text-gray-900">
+                    {% for type in school.types %}
+                      <span class="inline-block bg-gray-100 rounded-full px-2 py-1 text-xs font-medium text-gray-800 mr-1 mb-1">{{ type }}</span>
+                    {% endfor %}
+                  </td>
+                </tr>
+            {% endfor %}
+        </tbody>
+      </table>
+    </div>
+  </div>
+  {% endif %}
+{% endblock after_form %}

bullet/bullet_admin/urls/__init__.py

@@ -266,6 +266,11 @@
         education.SchoolCreateView.as_view(),
         name="school_create",
     ),
+    path(
+        "education/schools/import/",
+        education.SchoolCSVImportView.as_view(),
+        name="school_csv_import",
+    ),
     path("gallery/albums/", album.AlbumListView.as_view(), name="album_list"),
     path("gallery/albums/new/", album.AlbumCreateView.as_view(), name="album_create"),
     path(

bullet/bullet_admin/views/education.py

@@ -1,12 +1,12 @@
 from django.contrib import messages
 from django.http import HttpResponseRedirect
 from django.urls import reverse, reverse_lazy
-from django.views.generic import CreateView, ListView, UpdateView
+from django.views.generic import CreateView, FormView, ListView, UpdateView
 from education.models import School
 
 from bullet import search
 from bullet_admin.access import PermissionCheckMixin, is_country_admin
-from bullet_admin.forms.education import SchoolForm
+from bullet_admin.forms.education import SchoolCSVImportForm, SchoolForm
 from bullet_admin.mixins import MixinProtocol, RedirectBackMixin
 from bullet_admin.utils import get_active_competition, get_allowed_countries
 from bullet_admin.views import GenericForm
@@ -26,7 +26,10 @@ def get_queryset(self):
 class SchoolListView(PermissionCheckMixin, SchoolQuerySetMixin, GenericList, ListView):
     required_permissions = [is_country_admin]
 
-    list_links = [NewLink("school", reverse_lazy("badmin:school_create"))]
+    list_links = [
+        NewLink("school", reverse_lazy("badmin:school_create")),
+        Link("green", "mdi:upload", "Import", reverse_lazy("badmin:school_csv_import")),
+    ]
     table_fields = ["name", "address", "country"]
     table_field_templates = {
         "name": "bullet_admin/education/field__school_name.html",
@@ -78,7 +81,6 @@ class SchoolUpdateView(
     form_class = SchoolForm
     template_name = "bullet_admin/education/school_form.html"
     form_title = "Edit school"
-    require_unlocked_competition = False
     default_success_url = reverse_lazy("badmin:school_list")
 
     def get_form_kwargs(self):
@@ -106,7 +108,6 @@ class SchoolCreateView(
     CreateView,
 ):
     required_permissions = [is_country_admin]
-    require_unlocked_competition = False
     form_class = SchoolForm
     form_title = "New school"
     default_success_url = reverse_lazy("badmin:school_list")
@@ -126,3 +127,64 @@ def form_valid(self, form):
 
         messages.success(self.request, "School saved.")
         return HttpResponseRedirect(self.get_success_url())
+
+
+class SchoolCSVImportView(
+    PermissionCheckMixin,
+    RedirectBackMixin,
+    GenericForm,
+    FormView,
+):
+    required_permissions = [is_country_admin]
+    form_class = SchoolCSVImportForm
+    form_title = "Import schools"
+    form_multipart = True
+    template_name = "bullet_admin/education/school_csv_import.html"
+    default_success_url = reverse_lazy("badmin:school_list")
+
+    def get_form_kwargs(self):
+        kw = super().get_form_kwargs()
+        kw["competition"] = get_active_competition(self.request)
+        kw["user"] = self.request.user
+        return kw
+
+    def form_valid(self, form):
+        if form.cleaned_data["preview"]:
+            try:
+                schools_data = form.parse_csv()
+                return self.render_to_response(
+                    self.get_context_data(
+                        form=form,
+                        preview_data=schools_data[:10],
+                    )
+                )
+            except Exception as e:
+                messages.error(self.request, str(e))
+                return self.form_invalid(form)
+        else:
+            try:
+                result = form.import_schools()
+
+                if result["errors"]:
+                    messages.warning(
+                        self.request,
+                        f"Import completed with {len(result['errors'])} errors. "
+                        f"Created: {result['created']}, Updated: {result['updated']}",
+                    )
+                    for error in result["errors"][:5]:  # Show first 5 errors
+                        messages.error(self.request, error)
+                    if len(result["errors"]) > 5:
+                        messages.error(
+                            self.request,
+                            f"... and {len(result['errors']) - 5} more errors",
+                        )
+                else:
+                    messages.success(
+                        self.request,
+                        f"Successfully imported {result['created']} new schools and updated {result['updated']} existing schools.",
+                    )
+
+                return HttpResponseRedirect(self.get_success_url())
+            except Exception as e:
+                messages.error(self.request, str(e))
+                return self.form_invalid(form)

bullet/education/models.py

@@ -5,6 +5,7 @@
 
 
 class SchoolType(models.Model):
+    id: int
     name = models.CharField(max_length=64)
     note = models.CharField(
         max_length=32, help_text="shown in admin interface", blank=True
@@ -48,6 +49,7 @@ def __str__(self):
 
 
 class School(models.Model):
+    id: int
     name = models.CharField(max_length=256)
     types = models.ManyToManyField(SchoolType)
     address = models.CharField(max_length=256, blank=True, null=True)
@@ -74,13 +76,17 @@ class Meta:
     def __str__(self):
         return f"{self.name}, {self.address}"
 
-    def save(self, send_to_search=True, **kwargs):
-        x = super().save(**kwargs)
-        if send_to_search and search.enabled and not self.is_legacy:
+    def send_to_search(self):
+        if search.enabled and not self.is_legacy:
             search.client.index("schools").add_documents(
                 [self.for_search()],
                 "id",
             )
+
+    def save(self, send_to_search=True, **kwargs):
+        x = super().save(**kwargs)
+        if send_to_search:
+            self.send_to_search()
         return x
 
     def for_search(self):

bullet/education/tasks.py

@@ -0,0 +1,10 @@
+from django_rq import job
+
+from education.models import School
+
+
+@job
+def send_schools_to_search(school_ids: list[int]):
+    schools = School.objects.filter(id__in=school_ids)
+    for school in schools:
+        school.send_to_search()