Set all the species as unvalidated when uploaded (#4411)

Author: dimasciput

bims/migrations/0450_sitesetting_auto_validate_taxa_on_upload.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.16 on 2025-01-20 12:54
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bims', '0449_alter_biologicalcollectionrecord_uuid'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='sitesetting',
+            name='auto_validate_taxa_on_upload',
+            field=models.BooleanField(default=True, help_text='If True, taxa from CSV uploads are automatically validated.'),
+        ),
+    ]

bims/models/site_setting.py

@@ -31,6 +31,11 @@ class SiteSetting(Preferences):
                   '"filter_values": ["bims"]}]'
     )
 
+    auto_validate_taxa_on_upload = models.BooleanField(
+        default=True,
+        help_text='If True, taxa from CSV uploads are automatically validated.'
+    )
+
     default_location_site_cluster = models.CharField(
         max_length=100,
         help_text='SQL view name of the location site cluster which '

bims/scripts/taxa_upload.py

@@ -17,26 +17,48 @@
     IUCNStatus,
     IUCN_CATEGORIES,
     VernacularName,
-    ORIGIN_CATEGORIES, TaxonTag, SourceReference, SourceReferenceBibliography,
+    ORIGIN_CATEGORIES, TaxonTag, SourceReference,
+    SourceReferenceBibliography,
     Invasion
 )
 from bims.templatetags import is_fada_site
 from bims.utils.fetch_gbif import (
     fetch_all_species_from_gbif, fetch_gbif_vernacular_names
 )
 from bims.scripts.data_upload import DataCSVUpload
-from bims.utils.gbif import get_vernacular_names
 from td_biblio.exceptions import DOILoaderError
 from td_biblio.models import Entry
 from td_biblio.utils.loaders import DOILoader
 
+from preferences import preferences
+
 logger = logging.getLogger('bims')
 
 
 class TaxaProcessor(object):
 
     all_keys = {}
 
+    def add_taxon_to_taxon_group(self, taxonomy: Taxonomy, taxon_group: TaxonGroup, validated = True):
+        """
+        Add or update the relationship between a taxonomy and a taxon group,
+        ensuring the 'is_validated' field is properly set in both the
+        intermediate table (TaxonGroupTaxonomy) and on the Taxonomy object.
+        """
+        taxon_group.taxonomies.add(
+            taxonomy,
+            through_defaults={
+                'is_validated': validated
+            }
+        )
+
+    def add_taxon_to_taxon_group_unvalidated(self, taxonomy, taxon_group):
+        """
+        A helper function that calls `add_taxon_to_taxon_group` with
+        validated=True
+        """
+        self.add_taxon_to_taxon_group(taxonomy, taxon_group, validated=True)
+
     def handle_error(self, row, message):
         pass
 
@@ -647,12 +669,7 @@ def process_data(self, row, taxon_group: TaxonGroup):
                 )
 
                 # -- Add to taxon group
-                taxon_group.taxonomies.add(
-                    taxonomy,
-                    through_defaults={
-                        'is_validated': True
-                    }
-                )
+                self.add_taxon_to_taxon_group_unvalidated(taxonomy, taxon_group)
 
                 # -- Endemism
                 endemism = self.endemism(row)
@@ -746,8 +763,6 @@ def process_data(self, row, taxon_group: TaxonGroup):
                 if accepted_taxon:
                     taxonomy.accepted_taxonomy = accepted_taxon
 
-                taxonomy.validated = True
-
                 taxonomy.save()
                 self.finish_processing_row(row, taxonomy)
         except Exception as e:  # noqa
@@ -757,6 +772,14 @@ def process_data(self, row, taxon_group: TaxonGroup):
 class TaxaCSVUpload(DataCSVUpload, TaxaProcessor):
     model_name = 'taxonomy'
 
+    def add_taxon_to_taxon_group_unvalidated(self, taxonomy, taxon_group):
+        """
+        A helper function that calls `add_taxon_to_taxon_group` with
+        validated=False
+        """
+        auto_validate = preferences.SiteSetting.auto_validate_taxa_on_upload
+        self.add_taxon_to_taxon_group(taxonomy, taxon_group, validated=auto_validate)
+
     def finish_processing_row(self, row, taxonomy):
         # -- Add to taxon group
         taxon_group = self.upload_session.module_group

bims/tests/data/taxa_upload_family_2.csv

@@ -0,0 +1,2 @@
+Invasion,Taxon Rank,Vernacular name(s),Language,Kingdom,Phylum,Class,Order,Family,Genus,Species,SubSpecies,Taxon,Accepted Taxon,Taxonomic status,Scientific name and authority,Common NAME,Origin,Endemism,Conservation status global,Conservation status national,lentic,On GBIF,Lakes (Y/N),Author(s),ANT,AT
+Category 1a invasive,Class,,,Animalia,Annelida,Oligochaeta,,,,,,Oligochaeta2,,accepted,Oligochaeta2,"Earthworm (eng), Earthworm2, ミミズ (jpn)",Native,Unknown,Not evaluated,,,No,,,,

bims/tests/test_source_references.py

@@ -4,6 +4,7 @@
 from unittest.mock import patch, MagicMock
 
 from django_tenants.test.cases import FastTenantTestCase
+from datetime import datetime
 
 from bims.factories import EntryFactory
 from bims.tasks import get_source_reference_filter, generate_source_reference_filter
@@ -67,5 +68,5 @@ def test_generate_source_reference_filter(self, mock_get_current, mock_set, mock
             f'source_reference_filter_{self.tenant}',
             [
                 {'id': self.references[0].id, 'reference': f'- | {self.entry.publication_date.year} | Test', 'type': 'Peer-reviewed scientific article'},
-                {'id': self.references[1].id, 'reference': f'- | 2024 | {self.document.source}', 'type': 'Published book, report or thesis'}]
+                {'id': self.references[1].id, 'reference': f'- | {datetime.now().year} | {self.document.source}', 'type': 'Published book, report or thesis'}]
             , timeout=None)

bims/tests/test_taxa_upload.py

@@ -172,6 +172,35 @@ def test_taxa_upload(self, mock_fetch_all_species_from_gbif, mock_finish):
             ).exists()
         )
 
+    @mock.patch('bims.scripts.data_upload.DataCSVUpload.finish')
+    @mock.patch('bims.scripts.taxa_upload.fetch_all_species_from_gbif')
+    @mock.patch('bims.scripts.taxa_upload.preferences')
+    def test_taxa_upload_unvalidated(self, mock_preferences, mock_fetch_all_species_from_gbif, mock_finish):
+        mock_preferences.SiteSetting.auto_validate_taxa_on_upload = False
+        mock_finish.return_value = None
+        mock_fetch_all_species_from_gbif.return_value = self.taxonomy
+
+        with open(os.path.join(
+                test_data_directory, 'taxa_upload_family_2.csv'
+        ), 'rb') as file:
+            upload_session = UploadSessionF.create(
+                uploader=self.owner,
+                process_file=File(file),
+                module_group=self.taxon_group
+            )
+
+        taxa_csv_upload = TaxaCSVUpload()
+        taxa_csv_upload.upload_session = upload_session
+        taxa_csv_upload.start('utf-8-sig')
+
+        self.assertTrue(
+            TaxonGroupTaxonomy.objects.filter(
+                taxonomy__canonical_name='Oligochaeta2',
+                taxongroup=self.taxon_group,
+                is_validated=False
+            ).exists()
+        )
+
     @mock.patch('bims.scripts.data_upload.DataCSVUpload.finish')
     @mock.patch('bims.scripts.taxa_upload.fetch_all_species_from_gbif')
     def test_taxa_upload_variety_and_forma(self, mock_fetch_all_species_from_gbif, mock_finish):