SS-1953 New field for keywords from controlled vocabularies (#545)

Author: akochari

apps/background_tasks/tasks/doi_provisioning.py

@@ -61,20 +61,20 @@ def _build_additional_metadata(
         additional_metadata["creators"] = creators
         logger.debug(f"DOI provisioning: Added {len(creators)} creators from form data")
 
-    # Subjects/tags from form data (prefer form data over model instance tags)
+    # Subjects/keywords from form data (prefer form data over model instance subjects_keywords)
     if subjects and isinstance(subjects, list):
-        # Form-processed subjects/tags take priority
+        # Form-processed subjects/keywords take priority
         additional_metadata["subjects"] = subjects
         logger.debug(f"DOI provisioning: Added {len(subjects)} subjects from form data")
-    elif hasattr(app_instance, "tags") and app_instance.tags:
-        # Fallback to model instance tags if no form data
+    elif hasattr(app_instance, "subjects_keywords"):
+        # Fallback to model instance subjects_keywords if no form data
         try:
-            tag_names = [t.name for t in app_instance.tags.all()]
-            if tag_names:
-                additional_metadata["subjects"] = tag_names
-                logger.debug(f"DOI provisioning: Added {len(tag_names)} subjects from model instance")
+            subjects_keywords = app_instance.subjects_keywords or []
+            if subjects_keywords:
+                additional_metadata["subjects"] = subjects_keywords
+                logger.debug(f"DOI provisioning: Added {len(subjects_keywords)} subjects from model instance")
         except Exception:
-            tag_names = None
+            subjects_keywords = None
 
     return additional_metadata if additional_metadata else None
 
@@ -135,7 +135,7 @@ def execute(self, app_instance, **kwargs) -> dict[str, Any]:
             language=kwargs.get("language"),
             funding=kwargs.get("funding"),
             creators=kwargs.get("creators"),
-            subjects=kwargs.get("tags"),  # Note: task receives 'tags' but function expects 'subjects'
+            subjects=kwargs.get("subjects_keywords"),
         )
 
         try:

apps/forms/base.py

@@ -83,12 +83,12 @@ def _setup_form_fields(self):
 
         # Handle name
         self.fields["name"].initial = ""
-        # Initialize the tags field to existing tags or empty list
-        if self.instance and self.instance.pk and hasattr(self.instance, "tags"):
+        # Initialize subjects_keywords field to existing JSON data or empty list
+        if self.instance and self.instance.pk and hasattr(self.instance, "subjects_keywords"):
             self.instance.refresh_from_db()
-            self._original_tags = list(self.instance.tags.all())
+            self._original_subjects_keywords = self.instance.subjects_keywords or []
         else:
-            self._original_tags = []
+            self._original_subjects_keywords = []
 
         self._restore_model_help_text()
 
@@ -243,9 +243,44 @@ def clean_note_on_linkonly_privacy(self):
 
         return note_on_linkonly_privacy
 
-    def clean_tags(self):
-        cleaned_data = super().clean()
-        return cleaned_data.get("tags", [])
+    def clean_subjects_keywords(self):
+        raw = self.cleaned_data.get("subjects_keywords") or []
+
+        if isinstance(raw, str):
+            try:
+                raw = json.loads(raw)
+            except json.JSONDecodeError as e:
+                raise forms.ValidationError("Invalid subjects/keywords data.") from e
+
+        if not isinstance(raw, list):
+            raise forms.ValidationError("Subjects/keywords must be a list.")
+
+        cleaned_subjects_keywords = []
+
+        for item in raw:
+            if not isinstance(item, dict):
+                raise forms.ValidationError("Each subject/keyword entry must be an object.")
+
+            subject = (item.get("subject") or "").strip()
+            subject_scheme = (item.get("subject_scheme") or "").strip()
+            classification_code = (item.get("classification_code") or "").strip()
+
+            if not subject:
+                raise forms.ValidationError("Each subject/keyword entry must have a subject.")
+            if not subject_scheme:
+                raise forms.ValidationError("Each subject/keyword entry must have a scheme.")
+            if not classification_code:
+                raise forms.ValidationError("Each subject/keyword entry must have an identifier.")
+
+            cleaned_subjects_keywords.append(
+                {
+                    "subject": subject,
+                    "subject_scheme": subject_scheme,
+                    "classification_code": classification_code,
+                }
+            )
+
+        return cleaned_subjects_keywords
 
     def clean_funding_sources_json(self):
         raw = self.cleaned_data.get("funding_sources_json") or "[]"
@@ -372,26 +407,28 @@ def normalize(data):
                 # If there's an error, keep the field in changed_data to be safe
                 pass
 
-        # Handle tags field - compare current input with initial tags
-        if "tags" in changed_data and self.instance and self.instance.pk:
+        # Handle subjects_keywords field - compare JSON data
+        if "subjects_keywords" in changed_data and self.instance and self.instance.pk:
             try:
-                # Try invenio_tags field first, then tags field
-                current_tags_input = self.data.get("invenio_tags", "") or self.data.get("tags", "") or ""
-                initial_tags_input = None
-
-                if "invenio_tags" in self.fields:
-                    initial_tags_input = self.fields["invenio_tags"].initial or ""
-                elif hasattr(self, "_original_tags") and self._original_tags:
-                    # Fallback to database tags formatted as pipe-separated
-                    initial_tags_input = " | ".join(str(tag) for tag in self._original_tags)
-                else:
-                    initial_tags_input = ""
+                current_subjects_keywords = self.data.get("subjects_keywords", "") or "[]"
+                initial_subjects_keywords = self.fields["subjects_keywords"].initial or "[]"
 
-                # If the tag input is the same, remove from changed_data
-                if current_tags_input == initial_tags_input:
-                    changed_data.remove("tags")
-            except (AttributeError, KeyError):
-                # If there's an error, keep the field in changed_data to be safe
+                current_data = (
+                    json.loads(current_subjects_keywords)
+                    if isinstance(current_subjects_keywords, str)
+                    else current_subjects_keywords
+                )
+                initial_data = (
+                    json.loads(initial_subjects_keywords)
+                    if isinstance(initial_subjects_keywords, str)
+                    else initial_subjects_keywords
+                )
+
+                # If the data is the same, remove from changed_data
+                if current_data == initial_data:
+                    changed_data.remove("subjects_keywords")
+            except (json.JSONDecodeError, KeyError, ValueError, TypeError):
+                # If there's an error parsing, keep the field in changed_data to be safe
                 pass
 
         # Handle language field - compare current value with initial form value

apps/forms/custom.py

@@ -49,12 +49,12 @@ def __init__(self, *args, **kwargs):
 
     def _load_existing_tags_to_invenio_field(self):
         """Load existing database tags into the invenio_tags field"""
-        if self.instance and self.instance.pk and hasattr(self.instance, "tags"):
-            existing_tags = list(self.instance.tags.all())
+        if self.instance and self.instance.pk and hasattr(self.instance, "subjects_keywords"):
+            existing_tags = self.instance.subjects_keywords or []
             if existing_tags:
                 # Convert tag objects to pipe-separated format for template
-                tag_names = [str(tag) for tag in existing_tags]
-                tag_string = " | ".join(tag_names)
+                tag_names = [item.get("subject", "") for item in existing_tags if isinstance(item, dict)]
+                tag_string = " | ".join(filter(None, tag_names))
                 self.fields["invenio_tags"].initial = tag_string
             else:
                 self.fields["invenio_tags"].initial = ""
@@ -161,13 +161,13 @@ def clean(self):
 
             if invenio_tags_value:
                 # Process the tags using the existing cleaning logic
-                cleaned_data["tags"] = self.clean_keyword_tags()
-            elif self.instance and self.instance.pk and hasattr(self.instance, "tags"):
+                cleaned_data["subjects_keywords"] = self.clean_keyword_tags()
+            elif self.instance and self.instance.pk and hasattr(self.instance, "subjects_keywords"):
                 # If no tags in form but instance has existing tags, preserve them
-                existing_tags = list(self.instance.tags.all())
+                existing_tags = self.instance.subjects_keywords or []
                 if existing_tags:
                     # Keep existing tags as they are
-                    cleaned_data["tags"] = existing_tags
+                    cleaned_data["subjects_keywords"] = existing_tags
         return cleaned_data
 
     class Meta:
@@ -183,11 +183,11 @@ class Meta:
             "source_code_url",
             "port",
             "image",
-            "tags",
+            "subjects_keywords",
             "default_url_subpath",
             "mount_path",
         ]
         labels = {
             "note_on_linkonly_privacy": "Reason for choosing the link only option",
-            "tags": "Subjects and keywords",
+            "subjects_keywords": "Subjects and keywords",
         }

apps/forms/dash.py

@@ -48,12 +48,12 @@ def __init__(self, *args, **kwargs):
 
     def _load_existing_tags_to_invenio_field(self):
         """Load existing database tags into the invenio_tags field"""
-        if self.instance and self.instance.pk and hasattr(self.instance, "tags"):
-            existing_tags = list(self.instance.tags.all())
+        if self.instance and self.instance.pk and hasattr(self.instance, "subjects_keywords"):
+            existing_tags = self.instance.subjects_keywords or []
             if existing_tags:
                 # Convert tag objects to pipe-separated format for template
-                tag_names = [str(tag) for tag in existing_tags]
-                tag_string = " | ".join(tag_names)
+                tag_names = [item.get("subject", "") for item in existing_tags if isinstance(item, dict)]
+                tag_string = " | ".join(filter(None, tag_names))
                 self.fields["invenio_tags"].initial = tag_string
             else:
                 self.fields["invenio_tags"].initial = ""
@@ -168,13 +168,13 @@ def clean(self):
 
             if invenio_tags_value:
                 # Process the tags using the existing cleaning logic
-                cleaned_data["tags"] = self.clean_keyword_tags()
-            elif self.instance and self.instance.pk and hasattr(self.instance, "tags"):
+                cleaned_data["subjects_keywords"] = self.clean_keyword_tags()
+            elif self.instance and self.instance.pk and hasattr(self.instance, "subjects_keywords"):
                 # If no tags in form but instance has existing tags, preserve them
-                existing_tags = list(self.instance.tags.all())
+                existing_tags = self.instance.subjects_keywords or []
                 if existing_tags:
                     # Keep existing tags as they are
-                    cleaned_data["tags"] = existing_tags
+                    cleaned_data["subjects_keywords"] = existing_tags
         return cleaned_data
 
     class Meta:
@@ -188,10 +188,10 @@ class Meta:
             "source_code_url",
             "port",
             "image",
-            "tags",
+            "subjects_keywords",
             "default_url_subpath",
         ]
         labels = {
-            "tags": "Subjects and keywords",
+            "subjects_keywords": "Subjects and keywords",
             "note_on_linkonly_privacy": "Reason for choosing the link only option",
         }

apps/forms/depictio.py

@@ -26,12 +26,12 @@ def _setup_form_fields(self):
         )
 
         # Load existing tags into the invenio_tags field
-        if self.instance and self.instance.pk and hasattr(self.instance, "tags"):
-            existing_tags = list(self.instance.tags.all())
+        if self.instance and self.instance.pk and hasattr(self.instance, "subjects_keywords"):
+            existing_tags = self.instance.subjects_keywords or []
             if existing_tags:
                 # Convert tag objects to pipe-separated format for template
-                tag_names = [str(tag) for tag in existing_tags]
-                tag_string = " | ".join(tag_names)
+                tag_names = [item.get("subject", "") for item in existing_tags if isinstance(item, dict)]
+                tag_string = " | ".join(filter(None, tag_names))
                 self.fields["invenio_tags"].initial = tag_string
             else:
                 self.fields["invenio_tags"].initial = ""
@@ -78,13 +78,13 @@ def clean(self):
 
             if invenio_tags_value:
                 # Process the tags using the existing cleaning logic
-                cleaned_data["tags"] = self.clean_keyword_tags()
-            elif self.instance and self.instance.pk and hasattr(self.instance, "tags"):
+                cleaned_data["subjects_keywords"] = self.clean_keyword_tags()
+            elif self.instance and self.instance.pk and hasattr(self.instance, "subjects_keywords"):
                 # If no tags in form but instance has existing tags, preserve them
-                existing_tags = list(self.instance.tags.all())
+                existing_tags = self.instance.subjects_keywords or []
                 if existing_tags:
                     # Keep existing tags as they are
-                    cleaned_data["tags"] = existing_tags
+                    cleaned_data["subjects_keywords"] = existing_tags
 
         return cleaned_data
 
@@ -101,7 +101,7 @@ def changed_data(self):
 
     class Meta:
         model = DepictioInstance
-        fields = ["name", "description", "access", "tags"]
+        fields = ["name", "description", "access", "subjects_keywords"]
         labels = {
-            "tags": "Subjects and keywords",
+            "subjects_keywords": "Subjects and keywords",
         }

apps/forms/gradio.py

@@ -47,12 +47,12 @@ def __init__(self, *args, **kwargs):
 
     def _load_existing_tags_to_invenio_field(self):
         """Load existing database tags into the invenio_tags field"""
-        if self.instance and self.instance.pk and hasattr(self.instance, "tags"):
-            existing_tags = list(self.instance.tags.all())
+        if self.instance and self.instance.pk and hasattr(self.instance, "subjects_keywords"):
+            existing_tags = self.instance.subjects_keywords or []
             if existing_tags:
                 # Convert tag objects to pipe-separated format for template
-                tag_names = [str(tag) for tag in existing_tags]
-                tag_string = " | ".join(tag_names)
+                tag_names = [item.get("subject", "") for item in existing_tags if isinstance(item, dict)]
+                tag_string = " | ".join(filter(None, tag_names))
                 self.fields["invenio_tags"].initial = tag_string
             else:
                 self.fields["invenio_tags"].initial = ""
@@ -134,13 +134,13 @@ def clean(self):
 
             if invenio_tags_value:
                 # Process the tags using the existing cleaning logic
-                cleaned_data["tags"] = self.clean_keyword_tags()
-            elif self.instance and self.instance.pk and hasattr(self.instance, "tags"):
+                cleaned_data["subjects_keywords"] = self.clean_keyword_tags()
+            elif self.instance and self.instance.pk and hasattr(self.instance, "subjects_keywords"):
                 # If no tags in form but instance has existing tags, preserve them
-                existing_tags = list(self.instance.tags.all())
+                existing_tags = self.instance.subjects_keywords or []
                 if existing_tags:
                     # Keep existing tags as they are
-                    cleaned_data["tags"] = existing_tags
+                    cleaned_data["subjects_keywords"] = existing_tags
         return cleaned_data
 
     class Meta:
@@ -156,10 +156,10 @@ class Meta:
             "source_code_url",
             "port",
             "image",
-            "tags",
+            "subjects_keywords",
             "mount_path",
         ]
         labels = {
             "note_on_linkonly_privacy": "Reason for choosing the link only option",
-            "tags": "Subjects and keywords",
+            "subjects_keywords": "Subjects and keywords",
         }

apps/forms/mixins.py

@@ -425,14 +425,24 @@ def clean_keyword_tags(self):
 
         service = VocabularyMemoryService()
         valid_tags = []
+
         for tag in tags_list:
             found = False
+
             for term in service.term_metadata.values():
                 if (term.subject or "").lower() == tag.lower():
-                    valid_tags.append(tag)
+                    valid_tags.append(
+                        {
+                            "subject": term.subject,
+                            "subject_scheme": term.subject_scheme,
+                            "classification_code": term.classification_code,
+                        }
+                    )
                     found = True
                     break
+
             if not found:
                 self.add_error("invenio_tags", f"Tag '{tag}' is not valid.")
                 return []
+
         return valid_tags