Add gbif dataset filter (#5030)

* Add gbif dataset filter

* Cache datasetkeys

Author: dimasciput

bims/api_urls.py

@@ -40,6 +40,7 @@
     TaxonTagAutocompleteAPIView, AddTagAPIView, TaxonProposalDetail, IUCNStatusFetchView, TaxonTreeJsonView,
     HarvestIUCNStatus, ApproveTaxonGroupProposalsView, ClearTaxaNotAssociatedInTaxonGroup
 )
+from bims.api_views.dataset import DatasetAutocompleteAPIView
 from bims.api_views.cluster import ClusterList
 from bims.api_views.collection import (
     CollectionDownloader
@@ -380,6 +381,9 @@
     re_path(r'^taxon-tag-autocomplete/$',
             TaxonTagAutocompleteAPIView.as_view(),
             name='taxon-tag-autocomplete'),
+    re_path(r'^dataset-autocomplete/$',
+            DatasetAutocompleteAPIView.as_view(),
+            name='dataset-autocomplete'),
     path('taxonomy/<int:pk>/add-tag/',
          AddTagAPIView.as_view(),
          name='add-tag-taxon'),

bims/api_views/dataset.py

@@ -0,0 +1,64 @@
+from rest_framework.views import APIView
+from rest_framework.response import Response
+from django.core.cache import cache
+from bims.models.dataset import Dataset
+from bims.models.biological_collection_record import BiologicalCollectionRecord
+from bims.serializers.dataset_serializer import DatasetSerializer
+
+
+# Cache key for GBIF dataset keys
+GBIF_DATASET_KEYS_CACHE_KEY = 'gbif_dataset_keys'
+GBIF_DATASET_KEYS_CACHE_TIMEOUT = 86400  # 24 hours
+
+
+def clear_dataset_cache():
+    """
+    Clear the GBIF dataset keys cache.
+    Call this function after importing new GBIF data.
+    """
+    cache.delete(GBIF_DATASET_KEYS_CACHE_KEY)
+
+
+class DatasetAutocompleteAPIView(APIView):
+    """
+    Autocomplete API for GBIF datasets.
+
+    Query params:
+    - q: search term for dataset name
+    - ids: comma-separated database IDs for bootstrapping
+    """
+
+    def get(self, request, format=None):
+        query = request.query_params.get('q', '')
+        ids_param = request.query_params.get('ids', '')
+
+        dataset_keys = cache.get(GBIF_DATASET_KEYS_CACHE_KEY)
+
+        if dataset_keys is None:
+            dataset_keys = BiologicalCollectionRecord.objects.filter(
+                source_collection='gbif'
+            ).exclude(
+                dataset_key__isnull=True
+            ).values_list('dataset_key', flat=True).distinct()
+
+            dataset_keys = list(filter(None, set(dataset_keys)))
+            # Cache for 24 hours
+            cache.set(GBIF_DATASET_KEYS_CACHE_KEY, dataset_keys, GBIF_DATASET_KEYS_CACHE_TIMEOUT)
+
+        # Base queryset: only datasets with GBIF records
+        base_qs = Dataset.objects.filter(
+            uuid__in=dataset_keys
+        )
+
+        if ids_param:
+            # Bootstrap mode: fetch by database IDs
+            ids_list = [int(id.strip()) for id in ids_param.split(',') if id.strip()]
+            datasets = base_qs.filter(id__in=ids_list)
+        else:
+            # Search mode: filter by name
+            if len(query) < 2:
+                return Response([])
+            datasets = base_qs.filter(name__icontains=query).distinct()[:10]
+
+        serializer = DatasetSerializer(datasets, many=True)
+        return Response(serializer.data)

bims/api_views/search.py

@@ -366,6 +366,30 @@ def taxon_tags(self):
 
         return [t.strip() for t in raw_tags.split(',') if t.strip()]
 
+    @property
+    def dataset_keys(self):
+        """
+        Returns a list of dataset UUIDs to filter GBIF records by.
+        Converts dataset IDs from the URL to UUIDs.
+        """
+        from bims.models.dataset import Dataset
+
+        dataset_ids = self.parse_request_json('datasetKeys')
+        if not dataset_ids:
+            return []
+
+        # Convert dataset IDs to UUIDs
+        try:
+            dataset_ids = [int(id) for id in dataset_ids]
+            dataset_uuids = list(
+                Dataset.objects.filter(
+                    id__in=dataset_ids
+                ).values_list('uuid', flat=True)
+            )
+            return dataset_uuids
+        except (ValueError, TypeError):
+            return []
+
     @property
     def polygon(self):
         try:
@@ -669,6 +693,13 @@ def process_search(self):
                     source_collection_filters.append(source_collection)
         if source_collection_filters:
             filters['source_collection__in'] = source_collection_filters
+        if self.dataset_keys:
+            filters['dataset_key__in'] = self.dataset_keys
+            # Ensure GBIF is included when datasets are specified
+            if 'source_collection__in' not in filters:
+                filters['source_collection__in'] = ['gbif']
+            elif 'gbif' not in filters['source_collection__in']:
+                filters['source_collection__in'].append('gbif')
         if self.endemic:
             endemism_list = []
             for endemic in self.endemic:

bims/scripts/import_gbif_occurrences.py

@@ -939,5 +939,9 @@ def fetch_and_process_gbif_data(country_codes, geom_str=''):
     except Exception as e:
         log_to_file_or_logger(log_file_path, message=str(e))
         return str(e)
+    finally:
+        # Clear the dataset keys cache after GBIF import
+        from bims.api_views.dataset import clear_dataset_cache
+        clear_dataset_cache()
 
     return message

bims/serializers/dataset_serializer.py

@@ -0,0 +1,8 @@
+from rest_framework import serializers
+from bims.models.dataset import Dataset
+
+
+class DatasetSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = Dataset
+        fields = ['id', 'name', 'abbreviation']

bims/static/js/router.js

@@ -17,7 +17,7 @@ define(['backbone', 'views/olmap', 'utils/events_connector', 'shared'], function
                 'taxon', 'months', 'siteId', 'search', 'collector',
                 'category', 'yearFrom', 'yearTo', 'userBoundary', 'referenceCategory',
                 'boundary', 'reference', 'endemic', 'invasions', 'conservationStatus', 'spatialFilter',
-                'taxon', 'validated', 'modules', 'sourceCollection', 'ecologicalCategory',
+                'taxon', 'validated', 'modules', 'sourceCollection', 'datasetKeys', 'ecologicalCategory',
                 'module', 'tags', 'rank', 'orderBy', 'siteIdOpen', 'polygon',
                 'dst', 'ecosystemType'
             ];

bims/static/js/shared.js

@@ -12,7 +12,7 @@ define(['backbone', 'underscore', 'utils/storage', 'utils/color', 'utils/url', '
             "&reference=<%= reference %>&endemic=<%= endemic %>&invasions=<%= invasions %>&conservationStatus=<%= conservationStatus %>" +
             "&modules=<%= modules %>&validated=<%= validated %>&sourceCollection=<%= sourceCollection %>" +
             "&module=<%= module %>&ecologicalCategory=<%= ecologicalCategory %>&rank=<%= rank %>"+
-            "&tags=<%= tags %>" +
+            "&tags=<%= tags %>&datasetKeys=<%= datasetKeys %>" +
             "&siteIdOpen=<%= siteIdOpen %>&orderBy=<%= orderBy %>&polygon=<%= polygon %>&dst=<%= dst %>&ecosystemType=<%= ecosystemType %>",
         LocationSiteDetailXHRRequest: null,
         WetlandDashboardXHRRequest: null,