Merge pull request #29 from entrepreneur-interet-general/sp9-squash

Sp9 squash

Author: Ash-Crow

bnsp/admin.py

@@ -1,6 +1,6 @@
-from django.contrib import admin
-from django.utils.html import format_html
-from django.urls import reverse
+from django.contrib import admin, messages
+from django.http.response import HttpResponseRedirect
+from django.utils.translation import ngettext
 
 from francedata.services.django_admin import (
     TimeStampModelAdmin,
@@ -9,11 +9,11 @@
 
 from bnsp import models
 
-# Register your models here.
-
 
 @admin.register(models.Query)
 class QueryAdmin(TimeStampModelAdmin):
+    change_form_template = "bnsp/admin/query_changeform.html"
+
     search_fields = ("name", "query")
     list_display = (
         "__str__",
@@ -54,7 +54,35 @@ class QueryAdmin(TimeStampModelAdmin):
         ("Métadonnées", {"fields": ["id", "created_at", "updated_at"]}),
     ]
 
+    actions = ["force_run_queries"]
+
     def view_documents_link(self, obj):
         return view_reverse_changelink(obj, "core", "bnsp_query", "document")
 
     view_documents_link.short_description = "Documents"
+
+    # Force immediate execution of query on the admin detail view
+    def response_change(self, request, obj):
+        if "_force_run_query" in request.POST:
+            obj.run()
+            return HttpResponseRedirect(".")  # stay on the same detail page
+        return super().response_change(request, obj)
+
+    # Force immediate execution of queries on the admin list view
+    @admin.action(description="Lancer les requêtes maintenant")
+    def force_run_queries(self, request, queryset):
+        for q in queryset:
+            q.run()
+
+        updated = len(queryset)
+
+        self.message_user(
+            request,
+            ngettext(
+                "%d requête a été exécutée avec succès.",
+                "%d requêtes ont été exécutées avec succès.",
+                updated,
+            )
+            % updated,
+            messages.SUCCESS,
+        )

bnsp/migrations/0002_auto_20210909_1607.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.2.7 on 2021-09-09 14:07
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bnsp', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='query',
+            name='name',
+            field=models.CharField(max_length=255, unique=True, verbose_name='nom'),
+        ),
+        migrations.AlterField(
+            model_name='query',
+            name='query',
+            field=models.CharField(max_length=255, unique=True, verbose_name='requête'),
+        ),
+    ]

bnsp/models.py

@@ -1,3 +1,4 @@
+from dateutil import parser as dateparser
 from django.db import models
 from django.utils import timezone
 from django.utils.html import strip_tags
@@ -9,8 +10,8 @@
 
 class Query(TimeStampModel):
     # A query on the Gallica Search API
-    name = models.CharField(max_length=255, verbose_name="nom")
-    query = models.CharField(max_length=255, verbose_name="requête")
+    name = models.CharField(max_length=255, verbose_name="nom", unique=True)
+    query = models.CharField(max_length=255, verbose_name="requête", unique=True)
 
     source = models.ForeignKey(
         Source, on_delete=models.CASCADE, verbose_name="source associée"
@@ -50,10 +51,11 @@ def run(self, since: str = "") -> None:
 
         search = GallicaSearch(max_records=50)
         dated_query = f'({self.query}) and indexationdate > "{indexation_date}"'
-        response = search.get_records(dated_query)
+        search.fetch_records(dated_query)
 
-        if len(response):
-            for record in search.records.values():
+        records = search.get_records().values()
+        if len(records):
+            for record in records:
                 self.create_or_update_document(record)
             self.last_change = now
 
@@ -93,9 +95,12 @@ def create_or_update_document(self, record: Record) -> None:
         new_doc.title = strip_tags(record.title[:255])
 
         # Document vintage
-        year, _ = DataYear.objects.get_or_create(year=record.date)
-
-        new_doc.years.add(year)
+        try:
+            date = dateparser.parse(record.date).strftime("%Y")
+            year, _ = DataYear.objects.get_or_create(year=date)
+            new_doc.years.add(year)
+        except dateparser._parser.ParserError:
+            pass
 
         # The description
         new_doc.body = ", ".join(record.get_values("dc:subject"))

bnsp/services/gallica_search_api.py

@@ -1,5 +1,7 @@
 import logging
 import time
+from django.core.exceptions import ValidationError
+from django.core.validators import URLValidator
 import requests
 
 import xmltodict
@@ -9,14 +11,18 @@
 
 class GallicaSearch:
     # API defaults
-    API_ENDPOINT = "https://gallica.bnf.fr/SRU"
+    DEFAULT_API_ENDPOINT = "https://gallica.bnf.fr/SRU"
     DEFAULT_MAX_RECORDS = 15  # Max: 50
     DEFAULT_START_RECORD = 1
     DEFAULT_MAX_RETRIES = 3
 
-    def __init__(self, max_records: int = DEFAULT_MAX_RECORDS) -> None:
+    def __init__(
+        self,
+        max_records: int = DEFAULT_MAX_RECORDS,
+        endpoint: str = DEFAULT_API_ENDPOINT,
+    ) -> None:
         self.set_max_records(max_records)
-
+        self.set_api_endpoint(endpoint)
         self.raw_records = []
         self.total_records = 0
         self.records = {}
@@ -34,6 +40,16 @@ def set_max_records(self, max_records: int) -> None:
         else:
             self.max_records = self.DEFAULT_MAX_RECORDS
 
+    def set_api_endpoint(self, endpoint: str) -> None:
+        """
+        Changes the SRU API endpoint.
+
+        param:
+        - endpoint: the base URL of the SRU on a white-label version of Gallica
+          (ex: "https://nutrisco-patrimoine.lehavre.fr/SRU")
+        """
+        self.api_endpoint = endpoint
+
     def set_slow_mode(self, slow_mode: float = 0) -> None:
         """
         Set the value of the slow_mode parameter (defaults to zero)
@@ -60,14 +76,14 @@ def gallica_search_retrieve(
                 "query": query,
             }
 
-            response = requests.get(self.API_ENDPOINT, params=payload)
+            response = requests.get(self.api_endpoint, params=payload)
 
             retries = self.DEFAULT_MAX_RETRIES
             if response.status_code == 500:
                 while retries:
                     logging.warning(f"Error 500, retrying (retries: {retries})")
                     time.sleep(5)
-                    response = requests.get(self.API_ENDPOINT, params=payload)
+                    response = requests.get(self.DEFAULT_API_ENDPOINT, params=payload)
                     if response.status_code == 200:
                         break
                     else:
@@ -90,10 +106,7 @@ def count_records(self, query: str) -> dict:
 
         return {"total_records": self.total_records}
 
-    def get_records(
-        self,
-        query: str,
-    ) -> dict:
+    def fetch_records(self, query: str) -> None:
         """
         Retrieve the full lists of records for a query.
         Calls the gallica_search_retrieve recursively until all results are retrieved.
@@ -134,41 +147,97 @@ def get_records(
         else:
             logging.info("The research returned no (new) results.")
 
-        return {"records": self.raw_records, "total_records": self.total_records}
-
     def parse_records(self) -> None:
         for raw in self.raw_records:
+            raw = dict(raw)
             record = Record(raw)
             self.records[record.ark_id] = record
 
+    def get_records(self) -> dict:
+        """
+        Returns the dict with the records
+        """
+        return self.records
+
 
 class Record:
     def __init__(self, raw: dict) -> None:
         self.raw = raw
-        self.get_ark()
+        self.set_ark()
+        self.set_date()
         self.ark_id = self.ark_url.split("/")[-1]
-        self.title = raw["dc:title"]
-        self.date = raw["dc:date"]
+        self.title = self.get_first_value("dc:title")
 
     def get_values(self, key: str) -> list:
         """
         A datapoint can either be a string or a list of strings.
 
         This function casts everything into a list to streamline the parsing.
+
+        It also returns an empty list if the key is missing in the record.
         """
-        values = self.raw[key]
-        if isinstance(values, str):
-            values = [values]
+        if key in self.raw:
+            values = self.raw[key]
+
+            if isinstance(values, str):
+                values = [values]
+        else:
+            values = []
 
         return values
 
-    def get_ark(self) -> str:
+    def get_first_value(self, key: str) -> str:
+        """
+        Returns the first value, or an empty string if no value is present
+        """
+        values = self.get_values(key)
+        if len(values):
+            return values[0]
+        else:
+            return ""
+
+    def set_date(self) -> None:
+        """
+        Stores the 'dc:date' value as a string
+        """
+        if "dc:date" in self.raw:
+            date = self.get_first_value("dc:date")
+
+            # Manage some common bad values
+            if date == "[S.d.]":
+                date = ""
+
+            if "à nos jours" in date:
+                date = ""
+
+            # If a date range is provided, only keep the latest year
+            if "-" in date:
+                date_range = date.split("-")
+                date = date_range[-1]
+            self.date = date
+
+        else:
+            self.date = ""
+
+    def set_ark(self, ark_root="https://gallica.bnf.fr/ark") -> str:
         ids = self.get_values("dc:identifier")
 
+        self.ark_url = ""
+        # First, check for the proper Gallica ark
         for id in ids:
-            if "https://gallica.bnf.fr/ark" in id:
+            if ark_root in id:
                 self.ark_url = id
 
+        # Else, check for any URL
+        if not self.ark_url:
+            url_validate = URLValidator()
+            for id in ids:
+                try:
+                    url_validate(id)
+                    self.ark_url = id
+                except ValidationError:
+                    pass
+
         return self.ark_url
 
     def get_thumbnail(self) -> str:

bnsp/templates/bnsp/admin/query_changeform.html

@@ -0,0 +1,8 @@
+{% extends 'admin/change_form.html' %}
+
+{% block submit_buttons_bottom %}
+    <div class="submit-row">
+        <input type="submit" value="Lancer la requête maintenant" name="_force_run_query">
+    </div>
+    {{ block.super }}
+{% endblock %}