Merge branch 'main' of github.com:entrepreneur-interet-general/opencollectivites into main

Author: Ash-Crow

.github/workflows/django.yml

@@ -0,0 +1,44 @@
+name: Django-OpenCollectivites CI
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      max-parallel: 4
+      matrix:
+        python-version: [3.9]
+    services:
+      postgres:
+        image: postgres:10.8
+        env:
+          POSTGRES_USER: sample_user_name
+          POSTGRES_PASSWORD: sample_password
+          POSTGRES_DB: sample_db_name
+        ports:
+          - 5432:5432
+        # needed because the postgres container does not provide a healthcheck
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: 🐍 Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: 🐘 psycopg2 prerequisites
+      run: sudo apt install libpq-dev
+    - name: 👷 Install Dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install poetry
+        poetry install
+    - name: 🧪 Run Tests
+      run: |
+        poetry run python manage.py test

INSTALL.md

@@ -32,7 +32,7 @@ postgres=# \c <db_name>
 - `git clone` ce répertoire quelque part et `cd` dedans.
 
 ### Régler les paramètres locaux
-- `cp settings_local.py.sample settings_local.py`
+- `cp settings_local_sample.py settings_local.py`
 - Remplir le fichier `settings_local.py`
 
 ### Créer et activer l'environnement virtuel et installer les dépendances

README.md

@@ -2,8 +2,9 @@
 [![Made with Wagtail](https://img.shields.io/badge/Made%20with-Wagtail-0F7676.svg)](https://wagtail.io/)
 [![GitHub license](https://img.shields.io/github/license/entrepreneur-interet-general/opencollectivites.svg)](https://github.com/entrepreneur-interet-general/opencollectivites/blob/master/LICENSE)
 [![Website open-collectivites.fr](https://img.shields.io/website.svg?down_color=red&down_message=down&up_color=green&up_message=up&url=https%3A%2F%2Fwww.open-collectivites.fr)](https://www.open-collectivites.fr)
-[![Sonarcloud Status](https://sonarcloud.io/api/project_badges/measure?project=entrepreneur-interet-general_opencollectivites&metric=alert_status)](https://sonarcloud.io/dashboard?id=entrepreneur-interet-general_opencollectivites)
+[![OpenCollectivites CI](https://github.com/entrepreneur-interet-general/opencollectivites/actions/workflows/django.yml/badge.svg)](https://github.com/entrepreneur-interet-general/opencollectivites/actions/workflows/django.yml)
 [![CodeQL](https://github.com/entrepreneur-interet-general/opencollectivites/actions/workflows/codeql-analysis.yml/badge.svg)](https://github.com/entrepreneur-interet-general/opencollectivites/actions/workflows/codeql-analysis.yml)
+[![Sonarcloud Status](https://sonarcloud.io/api/project_badges/measure?project=entrepreneur-interet-general_opencollectivites&metric=alert_status)](https://sonarcloud.io/dashboard?id=entrepreneur-interet-general_opencollectivites)
 
 # Open Collectivités
 <img align="right" width="250" src="open-collectivites.png?raw=true" title="Open Collectivités" alt="" />

config/settings.py

@@ -10,12 +10,17 @@
 https://docs.djangoproject.com/en/3.1/ref/settings/
 """
 
-# Keep secrets and environment_specific variables in a separate file not using version control
-from settings_local import *
 from os import path
 import sys
 import logging
 
+# Keep secrets and environment_specific variables in a separate file not using version control
+try:
+    from settings_local import *
+except ImportError:
+    from settings_local_sample import *
+
+
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
 BASE_DIR = LOCAL_BASE_DIR
 

external_apis/admin.py

@@ -68,6 +68,7 @@ class OpenDataSoftQueryAdmin(TimeStampModelAdmin):
                     "identify_regions",
                     "identify_departements",
                     "identify_metropoles",
+                    "identify_years",
                 ]
             },
         ),

external_apis/migrations/0006_opendatasoftquery_identify_years.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.8 on 2021-10-18 09:16
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('external_apis', '0005_alter_bnspquery_options'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='opendatasoftquery',
+            name='identify_years',
+            field=models.BooleanField(default=False, verbose_name='Identifier les années de publication'),
+        ),
+    ]

external_apis/models.py

@@ -63,6 +63,9 @@ class OpenDataSoftQuery(ExternalApiQuery):
     identify_metropoles = models.BooleanField(
         verbose_name="Identifier les noms de métropoles", default=False
     )
+    identify_years = models.BooleanField(
+        verbose_name="Identifier les années de publication", default=False
+    )
 
     class Meta:
         verbose_name = "OpenDataSoft — requête"
@@ -151,6 +154,9 @@ def create_or_update_document(self, result: dict) -> None:
         if self.identify_metropoles:
             new_doc.identify_metropoles(full_text)
 
+        if self.identify_years:
+            self.get_years_from_keywords(new_doc, metadata["keyword"])
+
     def apply_mappings(self, document: Document, metadata: dict) -> None:
         """
         Applies the selected mappings to the document
@@ -171,27 +177,37 @@ def add_metadata_with_mapping(
             metadata_values = metadata[field]
             for mapping_value, properties in mapping_values.items():
                 if mapping_value in metadata_values:
-                    if isinstance(properties, list):
-                        for property in properties:
-                            self.add_metadata_properties(document, property)
-                    else:
-                        self.add_metadata_properties(document, properties)
+                    self.add_metadata_properties(document, properties)
 
     def add_metadata_properties(self, document: Document, properties: dict) -> None:
         """
         Add metadata from a properties dict
         """
-        if properties["type"] == "topic":
-            document.topics.add(properties["value"])
-        elif properties["type"] == "scope":
-            document.scope.add(properties["value"])
-        elif properties["type"] == "publication_pages":
-            document.publication_pages.add(properties["value"])
-        elif properties["type"] == "region":
-            document.regions.add(properties["value"])
-        elif properties["type"] == "departement":
-            document.departements.add(properties["value"])
+        for key, values in properties.items():
+            if not isinstance(values, list):
+                values = [values]
+
+            if key == "topic":
+                document.topics.add(*values)
+            elif key == "scope":
+                document.scope.add(*values)
+            elif key == "publication_pages":
+                document.publication_pages.add(*values)
+            elif key == "region":
+                document.regions.add(*values)
+            elif key == "departement":
+                document.departements.add(*values)
+
+        document.save()
 
+    def get_years_from_keywords(self, document: Document, keywords: dict) -> None:
+        """
+        Tries to identify publications years from the keywords dict
+        """
+        for keyword in keywords:
+            if keyword.isnumeric() and int(keyword) >= 2000 and int(keyword) <= 2100:
+                year, year_is_new = DataYear.objects.get_or_create(year=keyword)
+                document.years.add(year)
         document.save()
 
 

external_apis/services/opendatasoft_api.py

@@ -42,9 +42,9 @@ def api_call(self, endpoint: str, payload: dict) -> dict:
             while retries:
                 logging.warning(f"An error happened, retrying (retries: {retries})")
                 logging.debug(f"Parameters used:")
-                logging.debug(f"endpoint:", endpoint)
-                logging.debug(f"payload:", payload)
-                logging.debug(f"response:", response.text)
+                logging.debug(f"endpoint: {endpoint}")
+                logging.debug(f"payload: {payload}")
+                logging.debug(f"response: {response.text}")
                 time.sleep(5)
                 response = requests.get(endpoint, params=payload)
                 if response.status_code == 200:
@@ -81,7 +81,9 @@ def api_call_loop(self, endpoint: str, payload: dict, results_key: str) -> list:
             logging.info("The research returned no (new) results.")
             return []
 
-    def catalog_datasets(self, where: str = "", sort: str = "", exclude: str = "") -> list:
+    def catalog_datasets(
+        self, where: str = "", sort: str = "", exclude: str = ""
+    ) -> list:
         """
         Perform a query on the Search endpoint
         """

external_apis/tests/test_services_gallica_search_api.py

@@ -15,10 +15,10 @@ def setUp(self) -> None:
             "?operation=searchRetrieve&version=1.2&maximumRecords=15"
             "&startRecord=1&query=dc.title+all+%22jaune+bleu%22"
         )
-        with open("bnsp/tests/testdata/SRU_jaune_bleu_p1.xml", "r") as f:
+        with open("external_apis/tests/testdata/SRU_jaune_bleu_p1.xml", "r") as f:
             self.test_response_p1 = f.read()
 
-        with open("bnsp/tests/testdata/SRU_jaune_bleu_p2.xml", "r") as f:
+        with open("external_apis/tests/testdata/SRU_jaune_bleu_p2.xml", "r") as f:
             self.test_response_p2 = f.read()
 
     def test_search_instance_is_created(self) -> None: