Merge branch 'main' into batbot-origsr

Author: BryonLewis

bats_ai/core/fixtures/species.json

@@ -1623,5 +1623,18 @@
     },
     "model": "core.Species",
     "pk": "133"
+  },
+  {
+    "fields": {
+      "category": "multiple",
+      "common_name": "Florida bonneted bat, Brazilian free-tailed bat",
+      "family": "",
+      "genus": "",
+      "species": "Eumops floridanus, Tadarida brasiliensis",
+      "species_code": "EUFLTABR",
+      "species_code_6": ""
+    },
+    "model": "core.Species",
+    "pk": 134
   }
 ]

bats_ai/core/management/commands/loadGRTS.py

@@ -37,7 +37,7 @@
         14,
         "CONUS",
         # Backup URL
-        "https://data.kitware.com/api/v1/item/697cc601e7dea9be44ec5aee/download",
+        "https://data.kitware.com/api/v1/item/6a05c0aef88b2b3dd9854751/download",
     ),
     # Optional regions (disabled by default). Include via --locations.
     "AKCAN": (
@@ -64,6 +64,12 @@
         "Puerto Rico",
         None,
     ),
+    "CONUS_OFFSHORE": (
+        "https://data.kitware.com/api/v1/item/6a05c0dbf88b2b3dd9854754/download",
+        24,
+        "Offshore CONUS",
+        None,
+    ),
 }
 
 LOCATION_ID_TO_KEY = {str(config[1]): key for key, config in LOCATION_SOURCES.items()}
@@ -122,7 +128,11 @@ def handle(self, *args, **options):  # noqa: C901, PLR0912, PLR0915
         shapefiles = [LOCATION_SOURCES[location_key] for location_key in selected_keys]
 
         for url, sample_frame_id, name, backup_url in shapefiles:
-            logger.info("Downloading shapefile for Location %s...", name)
+            logger.info(
+                "Downloading shapefile for Location %s with sample frame id %s",
+                name,
+                sample_frame_id,
+            )
             with tempfile.TemporaryDirectory() as tmpdir:
                 tmpdir = Path(tmpdir)
 
@@ -167,9 +177,20 @@ def handle(self, *args, **options):  # noqa: C901, PLR0912, PLR0915
                 else:
                     logger.warning("Shapefile CRS unknown; assuming EPSG:4326")
 
+                if "GRTS_ID" in gdf.columns:
+                    grts_id_col = "GRTS_ID"
+                elif "grts_id" in gdf.columns:
+                    grts_id_col = "grts_id"
+                    logger.info("GRTS_ID column not found; using grts_id for cell ids.")
+                else:
+                    raise CommandError(
+                        "Shapefile must contain a GRTS_ID or grts_id column; "
+                        f"columns present: {', '.join(map(str, gdf.columns))}"
+                    )
+
                 # Replace-on-reimport behavior: delete existing rows for any
                 # (grts_cell_id, sample_frame_id) present in this import batch.
-                incoming_grts_ids = set(gdf["GRTS_ID"].dropna().astype(int).tolist())
+                incoming_grts_ids = set(gdf[grts_id_col].dropna().astype(int).tolist())
                 if incoming_grts_ids:
                     deleted_count, _ = GRTSCells.objects.filter(
                         sample_frame_id=sample_frame_id,
@@ -191,11 +212,11 @@ def handle(self, *args, **options):  # noqa: C901, PLR0912, PLR0915
                     total=len(gdf),
                     desc=f"Importing {sample_frame_id}",
                 ):
-                    # Hard fail if GRTS_ID is missing
-                    if "GRTS_ID" not in row or row["GRTS_ID"] is None:
-                        raise CommandError(f"Row {idx} missing required GRTS_ID field!")
+                    # Hard fail if GRTS id is missing
+                    if grts_id_col not in row or row[grts_id_col] is None:
+                        raise CommandError(f"Row {idx} missing required {grts_id_col} field!")
 
-                    grts_id = int(row["GRTS_ID"])
+                    grts_id = int(row[grts_id_col])
                     cell_key = (grts_id, sample_frame_id)
                     if cell_key in seen_in_file:
                         continue
@@ -220,13 +241,13 @@ def handle(self, *args, **options):  # noqa: C901, PLR0912, PLR0915
 
                     if len(records_to_create) >= batch_size:
                         with transaction.atomic():
-                            GRTSCells.objects.bulk_create(records_to_create, ignore_conflicts=True)
+                            GRTSCells.objects.bulk_create(records_to_create)
                         records_to_create.clear()
 
                 # Insert remaining records
                 if records_to_create:
                     with transaction.atomic():
-                        GRTSCells.objects.bulk_create(records_to_create, ignore_conflicts=True)
+                        GRTSCells.objects.bulk_create(records_to_create)
 
                 logger.info(
                     "Finished importing shapefile for sample frame %s: %s new records",

bats_ai/core/utils/batbot_metadata.py

@@ -534,9 +534,7 @@ def generate_spectrogram_assets(
     )
     mask_paths, mask_origsr_paths = _split_origsr_paths(metadata.spectrogram.mask_path)
     masked_paths, masked_origsr_paths = _split_origsr_paths(metadata.spectrogram.masked_path)
-    waveplot_paths, waveplot_origsr_paths = _split_origsr_paths(
-        metadata.spectrogram.waveplot_path
-    )
+    waveplot_paths, waveplot_origsr_paths = _split_origsr_paths(metadata.spectrogram.waveplot_path)
     compressed_waveplot_paths, compressed_waveplot_origsr_paths = _split_origsr_paths(
         metadata.spectrogram.waveplot_compressed_path
     )

bats_ai/core/utils/grts_utils.py

@@ -1,5 +1,17 @@
 from __future__ import annotations
 
+from django.db.models import Case, F, IntegerField, Value, When
+
+
+def recording_effective_sample_frame_id_case() -> Case:
+    """Return a ``Case`` matching ``normalize_sample_frame_id`` for ``Recording`` rows."""
+    return Case(
+        When(sample_frame_id__isnull=True, then=Value(14)),
+        When(sample_frame_id=19, then=Value(20)),
+        default=F("sample_frame_id"),
+        output_field=IntegerField(),
+    )
+
 
 def normalize_sample_frame_id(sample_frame_id: int | None) -> int | None:
     """Normalize sample frame IDs for AKCAN compatibility.
@@ -12,4 +24,6 @@ def normalize_sample_frame_id(sample_frame_id: int | None) -> int | None:
     """
     if sample_frame_id == 19:
         return 20
+    if sample_frame_id is None:
+        return 14
     return sample_frame_id

bats_ai/core/views/recording.py

@@ -19,7 +19,6 @@
 from bats_ai.core.models import (
     Annotations,
     CompressedSpectrogram,
-    GRTSCells,
     PulseMetadata,
     Recording,
     RecordingAnnotation,
@@ -29,7 +28,7 @@
     Spectrogram,
 )
 from bats_ai.core.tasks.tasks import recording_compute_spectrogram
-from bats_ai.core.views.recording_location import _parse_bbox
+from bats_ai.core.views.recording_location import _parse_bbox, filter_recordings_by_map_bbox
 from bats_ai.core.views.species import SpeciesSchema
 
 if TYPE_CHECKING:
@@ -525,13 +524,7 @@ def get_recordings(  # noqa: C901
     if q.bbox and q.bbox.strip():
         min_lon, min_lat, max_lon, max_lat = _parse_bbox(q.bbox)
         bbox_poly = Polygon.from_bbox((min_lon, min_lat, max_lon, max_lat))
-        # Need to check the GRTSCells centroids as well as the recording_location
-        grts_cell_ids = GRTSCells.objects.filter(centroid_4326__intersects=bbox_poly).values_list(
-            "grts_cell_id", flat=True
-        )
-        queryset = queryset.filter(
-            Q(recording_location__intersects=bbox_poly) | Q(grts_cell_id__in=grts_cell_ids)
-        )
+        queryset = filter_recordings_by_map_bbox(queryset, bbox_poly)
 
     sort_field = q.sort_by or "created"
     order_prefix = "" if q.sort_direction == "asc" else "-"
@@ -610,13 +603,7 @@ def apply_filters_and_sort(qs: QuerySet[Recording]) -> QuerySet[Recording]:
         if bbox and bbox.strip():
             min_lon, min_lat, max_lon, max_lat = _parse_bbox(bbox)
             bbox_poly = Polygon.from_bbox((min_lon, min_lat, max_lon, max_lat))
-            # Need to check the GRTSCells centroids as well as the recording_location
-            grts_cell_ids = GRTSCells.objects.filter(
-                centroid_4326__intersects=bbox_poly
-            ).values_list("grts_cell_id", flat=True)
-            qs = qs.filter(
-                Q(recording_location__intersects=bbox_poly) | Q(grts_cell_id__in=grts_cell_ids)
-            )
+            qs = filter_recordings_by_map_bbox(qs, bbox_poly)
         order_prefix = "" if sort_direction == "asc" else "-"
         if sort_by == "owner_username":
             qs = qs.order_by(f"{order_prefix}owner__username")