GOATS-1357: Overhaul the management and listing of files in GOATS

Author: miguelgomezgomez

src/goats_cli/goats_template/{{ project_name }}/settings/base.py.jinja

@@ -439,5 +439,3 @@ GPP_ENV = "PRODUCTION"
 ANTARES_ENV = "PRODUCTION"
 
 ANTARES_TIMEOUT = 60 # Timeout for requests to ANTARES in seconds.
-
-

src/goats_tom/api_views/__init__.py

@@ -1,6 +1,7 @@
 from .antares2goats import Antares2GoatsViewSet
 from .astro_datalab import AstroDatalabViewSet
 from .base_recipe import BaseRecipeViewSet
+from .dataproduct_type import DataProductTypeViewSet
 from .dataproducts import DataProductsViewSet
 from .dragons_caldb import DRAGONSCaldbViewSet
 from .dragons_data import DRAGONSDataViewSet
@@ -33,6 +34,7 @@
     "DRAGONSProcessedFilesViewSet",
     "GPPViewSet",
     "DataProductsViewSet",
+    "DataProductTypeViewSet",
     "Antares2GoatsViewSet",
     "RunProcessorViewSet",
     "DRAGONSDataViewSet",

src/goats_tom/api_views/dataproduct_type.py

@@ -0,0 +1,34 @@
+"""Viewset for retagging the `data_product_type` of an existing `DataProduct`."""
+
+__all__ = ["DataProductTypeViewSet"]
+
+from django.conf import settings
+from guardian.shortcuts import get_objects_for_user
+from rest_framework import mixins, permissions
+from rest_framework.viewsets import GenericViewSet
+from tom_dataproducts.models import DataProduct
+
+from goats_tom.serializers import DataProductTypeUpdateSerializer
+
+
+class DataProductTypeViewSet(mixins.UpdateModelMixin, GenericViewSet):
+    """Allows updating just the `data_product_type` of a `DataProduct`.
+
+    Restricted to the data products the requesting user already has permission
+    to view, matching `tom_dataproducts.api_views.DataProductViewSet`.
+    """
+
+    queryset = DataProduct.objects.all()
+    serializer_class = DataProductTypeUpdateSerializer
+    permission_classes = [permissions.IsAuthenticated]
+    # Only PATCH is used by the frontend; don't expose an untested PUT.
+    http_method_names = ["patch", "head", "options"]
+
+    def get_queryset(self):
+        if settings.TARGET_PERMISSIONS_ONLY:
+            return super().get_queryset()
+        return get_objects_for_user(
+            self.request.user,
+            "tom_dataproducts.view_dataproduct",
+            klass=super().get_queryset(),
+        )

src/goats_tom/facilities/overrides.py

@@ -141,6 +141,26 @@ class UserAwareBLANCOSettings(UserTokenMixin, BLANCOSettings):
     credential_attr = "lcologin"
 
 
+class InferDataProductTypeMixin:
+    """
+    Infer and persist `data_product_type` for products saved without one.
+
+    TOMToolkit's generic `save_data_products` creates `DataProduct` rows with
+    an empty `data_product_type`; views like the visualizer filter on the
+    stored value, so tag them at ingestion instead of at display time.
+    """
+
+    def save_data_products(self, observation_record, product_id=None):
+        products = super().save_data_products(observation_record, product_id)
+        for dp in products:
+            if not dp.data_product_type and dp.data.name.endswith(
+                (".fits", ".fits.fz")
+            ):
+                dp.data_product_type = "fits_file"
+                dp.save(update_fields=["data_product_type"])
+        return products
+
+
 class UserAwareFacilityMixin:
     """
     Mixin for facility classes that ensures user-aware facility settings
@@ -202,23 +222,25 @@ def get_form_classes_for_display(self, **kwargs):
         return {key: self._wrap_form_class(cls) for key, cls in base_map.items()}
 
 
-class LCOFacility(UserAwareFacilityMixin, BaseLCOFacility):
+class LCOFacility(InferDataProductTypeMixin, UserAwareFacilityMixin, BaseLCOFacility):
     """
     LCO facility with per-user API keys.
     """
 
     settings_cls = UserAwareLCOSettings
 
 
-class SOARFacility(UserAwareFacilityMixin, BaseSOARFacility):
+class SOARFacility(InferDataProductTypeMixin, UserAwareFacilityMixin, BaseSOARFacility):
     """
     SOAR facility with per-user API keys.
     """
 
     settings_cls = UserAwareSOARSettings
 
 
-class BLANCOFacility(UserAwareFacilityMixin, BaseBLANCOFacility):
+class BLANCOFacility(
+    InferDataProductTypeMixin, UserAwareFacilityMixin, BaseBLANCOFacility
+):
     """
     BLANCO facility with per-user API keys.
     """

src/goats_tom/serializers/__init__.py

@@ -1,7 +1,7 @@
 from .antares2goats import Antares2GoatsSerializer
 from .astro_datalab import AstroDatalabSerializer
 from .base_recipe import BaseRecipeSerializer
-from .dataproduct import DataProductSerializer
+from .dataproduct import DataProductSerializer, DataProductTypeUpdateSerializer
 from .dataproduct_metadata import DataProductMetadataSerializer
 from .dragons_caldb import DRAGONSCaldbSerializer
 from .dragons_file import DRAGONSFileFilterSerializer, DRAGONSFileSerializer
@@ -33,6 +33,7 @@
     "BaseRecipeSerializer",
     "DRAGONSProcessedFilesSerializer",
     "DataProductSerializer",
+    "DataProductTypeUpdateSerializer",
     "RunProcessorSerializer",
     "DataProductMetadataSerializer",
     "Antares2GoatsSerializer",

src/goats_tom/serializers/dataproduct.py

@@ -13,13 +13,14 @@
 This custom serializer will only be used in DRAGONS reductions.
 """
 
-__all__ = ["DataProductSerializer"]
+__all__ = ["DataProductSerializer", "DataProductTypeUpdateSerializer"]
 
 from pathlib import Path
 
 from django.conf import settings
 from django.core.files.storage import default_storage
 from rest_framework import serializers
+from tom_dataproducts.models import DataProduct
 from tom_dataproducts.serializers import (
     DataProductSerializer as BaseDataProductSerializer,
 )
@@ -83,3 +84,15 @@ def create(self, validated_data):
         dp.save()
 
         return dp
+
+
+class DataProductTypeUpdateSerializer(serializers.ModelSerializer):
+    """Serializer to handle retagging a `DataProduct`'s `data_product_type`."""
+
+    data_product_type = serializers.ChoiceField(
+        choices=list(settings.DATA_PRODUCT_TYPES.keys())
+    )
+
+    class Meta:
+        model = DataProduct
+        fields = ["data_product_type"]

src/goats_tom/tasks/download_goa_files.py

@@ -23,6 +23,7 @@
 from goats_tom.utils import create_name_reduction_map
 
 logger = logging.getLogger(__name__)
+SPEC_FILE_SUFFIX = ("_1D", "_2D")
 
 
 @dramatiq.actor(
@@ -252,9 +253,12 @@ def download_goa_files(
 
             product_id = str(file_path.relative_to(settings.MEDIA_ROOT))
 
-            # Use the mapping to get the data product type.
-            # If not found, return default for calibration.
-            data_product_type = name_reduction_map.get(file_path.name, "fits_file")
+            # Spectra follow the _1D/_2D naming convention; otherwise use the
+            # reduction mapping, defaulting to calibration ("fits_file").
+            if file_path.stem.endswith(SPEC_FILE_SUFFIX):
+                data_product_type = "spectroscopy"
+            else:
+                data_product_type = name_reduction_map.get(file_path.name, "fits_file")
             # Query DataProduct by product_id.
             candidates = DataProduct.objects.filter(
                 product_id=product_id,

src/goats_tom/templates/partials/dataproduct_visualizer.html

@@ -90,4 +90,4 @@ <h5>Select and Plot Data Products</h5>
       noMatchesRow.style.display = visibleCount === 0 ? "" : "none";
     });
   });
-</script>
+</script>

src/goats_tom/templates/tom_dataproducts/partials/dataproduct_list_for_target.html

@@ -31,7 +31,6 @@ <h4>Data</h4>
             <th></th>
           </tr>
         </thead>
-        {% define_data_product_type products %}
         <tbody id="manageDataTBody">
           {% for product in products %}
           <tr data-file="{{ product.product_id }}">
@@ -81,7 +80,31 @@ <h4>Data</h4>
               >
             </td>
             <td>
-              {% if product.data_product_type %} {{ product.get_type_display }} {% endif %}
+              <div class="dropdown">
+                <button
+                  class="btn btn-secondary dropdown-toggle"
+                  type="button"
+                  data-bs-toggle="dropdown"
+                  aria-expanded="false">
+                  <span class="button-text">{{ product.data_product_type_label }}</span>
+                  <span
+                    class="spinner-border spinner-border-sm d-none"
+                    role="status"
+                    aria-hidden="true"></span>
+                </button>
+                <ul class="dropdown-menu" data-id="{{ product.id }}">
+                  {% for value, label in data_product_type_choices %}
+                  <li>
+                    <a
+                      class="dropdown-item {% if value == product.data_product_type %}active{% endif %}"
+                      href="#"
+                      data-value="{{ value }}"
+                      >{{ label }}</a
+                    >
+                  </li>
+                  {% endfor %}
+                </ul>
+              </div>
             </td>
             <!-- <td>
               {% if sharing_destinations %}
@@ -220,6 +243,78 @@ <h4>Data</h4>
       await actions[action]?.();
     });
 
+    // Event listener for handling data product type changes.
+    tableBody.addEventListener("click", async (event) => {
+      const item = event.target.closest("a.dropdown-item[data-value]");
+
+      if (!item) return;
+      event.preventDefault();
+
+      const menu = item.closest("ul.dropdown-menu");
+      const { id } = menu.dataset;
+      const { value } = item.dataset;
+      const label = item.textContent.trim();
+
+      const dropdown = item.closest(".dropdown");
+      const mainButton = dropdown.querySelector(".btn-secondary");
+      const buttonText = mainButton.querySelector(".button-text");
+      const spinner = mainButton.querySelector(".spinner-border");
+
+      buttonText.classList.add("d-none");
+      spinner.classList.remove("d-none");
+      mainButton.disabled = true;
+
+      try {
+        await updateDataProductType(id, value);
+        buttonText.textContent = label;
+        menu.querySelectorAll(".dropdown-item").forEach((dropdownItem) => {
+          dropdownItem.classList.toggle("active", dropdownItem === item);
+        });
+        window.toast?.show({
+          label: "Data Product Type Updated",
+          message: `Type changed to "${label}".`,
+          color: "success",
+        });
+      } catch (error) {
+        window.toast?.show({
+          label: "Failed to Update Data Product Type",
+          message: error.message,
+          color: "danger",
+        });
+      } finally {
+        buttonText.classList.remove("d-none");
+        spinner.classList.add("d-none");
+        mainButton.disabled = false;
+      }
+    });
+
+    /**
+     * Updates the `data_product_type` of a data product.
+     * @param {string} id - The unique identifier of the file.
+     * @param {string} dataProductType - The new data product type.
+     * @returns {Promise<void>} - Resolves when the request is completed.
+     */
+    const updateDataProductType = async (id, dataProductType) => {
+      const url = `/api/dataproducttype/${id}/`;
+
+      const response = await fetch(url, {
+        method: "PATCH",
+        headers: {
+          "Content-Type": "application/json",
+          "X-CSRFToken": "{{ csrf_token }}",
+        },
+        credentials: "same-origin",
+        body: JSON.stringify({ data_product_type: dataProductType }),
+      });
+      if (!response.ok) {
+        // The error response may not be JSON (e.g. an HTML error page).
+        const data = await response.json().catch(() => null);
+        throw new Error(
+          data?.data_product_type?.[0] || "Failed to update data product type."
+        );
+      }
+    };
+
     /**
      * Sends a file to Astro DataLab.
      * @param {string} id - The unique identifier of the file.

src/goats_tom/templatetags/dataproduct_visualizer.py

@@ -39,10 +39,6 @@ def dataproduct_visualizer(
 
     active_filter = Q(data_product_type=data_type)
 
-    if data_type == "spectroscopy":
-        active_filter |= Q(data_product_type="fits_file")
-        active_filter |= Q(data__iendswith="fits.fz")
-
     dataproducts = DataProduct.objects.filter(target=target).filter(active_filter)
 
     return {"target": target, "dataproducts": dataproducts, "data_type": data_type}