Add flag index_files_content to AIP profile and remove index from storage policy (#2436)

* Add flag index_files_content to AIP profile and remove index from storage policy

* Remove index field from storage policy serializers

* Add new db migration for storage policy

Author: henrikek

ESSArch_Core/configuration/admin.py

@@ -142,7 +142,6 @@ class StoragePolicyAdmin(NestedModelAdmin):
                 'validate_checksum',
                 'validate_xml',
                 'ingest_delete',
-                'index',
                 'receive_extract_sip',
                 'cache_minimum_capacity',
                 'cache_maximum_age',

ESSArch_Core/configuration/migrations/0034_remove_storagepolicy_index.py

@@ -0,0 +1,17 @@
+# Generated by Django 5.2.9 on 2025-12-04 10:13
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('configuration', '0033_alter_storagepolicy_index_and_more'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='storagepolicy',
+            name='index',
+        ),
+    ]

ESSArch_Core/configuration/models.py

@@ -200,16 +200,12 @@ class StoragePolicy(models.Model):
     )
 
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
-
-    index = models.BooleanField(default=False)
-
     cache_minimum_capacity = models.IntegerField(
         'Minimum size (bytes) available on cache before deleting content', default=0,
     )
     cache_maximum_age = models.IntegerField(
         'Maximum age (days) of content before deletion from cache, resets on access', default=0,
     )
-
     policy_id = models.CharField('Policy ID', max_length=32, unique=True)
     policy_name = models.CharField('Policy Name', max_length=255)
     policy_stat = models.BooleanField('Policy Status', default=True)

ESSArch_Core/configuration/serializers.py

@@ -221,7 +221,7 @@ def create(self, validated_data):
     class Meta:
         model = StoragePolicy
         fields = (
-            "id", "index",
+            "id",
             "cache_minimum_capacity", "cache_maximum_age",
             "policy_id", "policy_name",
             "policy_stat", "ais_project_name", "ais_project_id",
@@ -247,7 +247,7 @@ class Meta:
 class StoragePolicyNestedSerializer(StoragePolicySerializer):
     class Meta(StoragePolicySerializer.Meta):
         fields = (
-            "id", "index",
+            "id",
             "cache_minimum_capacity", "cache_maximum_age",
             "policy_id", "policy_name",
             "policy_stat", "ais_project_name", "ais_project_id",

ESSArch_Core/ip/models.py

@@ -1221,8 +1221,11 @@ def create_preservation_workflow(self):
 
         profile_type = self.get_package_type_display().lower()
         index_files = self.get_profile_data(profile_type).get('index_files', True)
+        index_files_content = self.get_profile_data(profile_type).get('index_files_content', True)
         index_cits = self.get_profile_data(profile_type).get('index_cits', True)
-        if index_files is True or index_files == 'True' or index_cits is True or index_cits == 'True':
+        if (index_files is True or index_files == 'True' or
+                index_files_content is True or index_files_content == 'True' or
+                index_cits is True or index_cits == 'True'):
             write_to_search_index = True
         else:
             write_to_search_index = False
@@ -2800,6 +2803,7 @@ def write_to_search_index(self, task):
                 indexed_files = ct_importer.import_content(cts, ip=self)
 
         index_files = self.get_profile_data(profile_type).get('index_files', True)
+        index_files_content = self.get_profile_data(profile_type).get('index_files_content', True)
         if index_files is True or index_files == 'True':
             group = None
             try:
@@ -2812,7 +2816,7 @@ def write_to_search_index(self, task):
             for root, dirs, files in walk(srcdir):
                 for d in dirs:
                     src = os.path.join(root, d)
-                    index_path(self, src, group=group)
+                    index_path(self, src, group=group, index_file_content=index_files_content)
 
                 for f in files:
                     src = os.path.join(root, f)
@@ -2821,7 +2825,7 @@ def write_to_search_index(self, task):
                         indexed_files.remove(src)
                     except ValueError:
                         # file has not been indexed, index it
-                        index_path(self, src, group=group)
+                        index_path(self, src, group=group, index_file_content=index_files_content)
 
         InformationPackageDocument.from_obj(self).save()
 

ESSArch_Core/search/ingest.py

@@ -21,15 +21,13 @@
 )
 
 
-def index_document(tag_version, filepath):
+def index_document(tag_version, filepath, index_file_content=True):
     logger = logging.getLogger('essarch.search.ingest')
     exclude_file_format_from_indexing_content = settings.EXCLUDE_FILE_FORMAT_FROM_INDEXING_CONTENT
 
     fid = FormatIdentifier()
     (format_name, format_version, format_registry_key) = fid.identify_file_format(filepath)
-    if format_registry_key not in exclude_file_format_from_indexing_content:
-        index_file_content = True
-    else:
+    if format_registry_key in exclude_file_format_from_indexing_content:
         index_file_content = False
 
     ip = tag_version.tag.information_package
@@ -84,7 +82,7 @@ def index_directory(tag_version, dirpath):
     return doc, tag_version
 
 
-def index_path(ip, path, parent=None, group=None):
+def index_path(ip, path, parent=None, group=None, index_file_content=True):
     """
     Indexes the file or directory at path to elasticsearch
 
@@ -113,7 +111,7 @@ def index_path(ip, path, parent=None, group=None):
         tag_version.elastic_index = 'document'
         # TODO: minimize db queries
         tag_version.type = TagVersionType.objects.get_or_create(name='document', archive_type=False)[0]
-        doc, tag_version = index_document(tag_version, path)
+        doc, tag_version = index_document(tag_version, path, index_file_content)
         tag_version.save()
     else:
         tag_version.elastic_index = 'directory'

ESSArch_Core/templates/eu/EU_AIP_v204.json

@@ -305,6 +305,23 @@
 				"popover-trigger": "'mouseenter'"
 			}
 		},
+		{
+			"key": "index_files_content",
+			"type": "select",
+			"defaultValue": true,
+			"hidden": false,
+			"templateOptions": {
+				"required": false,
+				"disabled": false,
+				"label": "Index files content",
+				"options": [{"name": "Yes","value": true},{"name": "No","value": false}]
+			},
+			"ngModelElAttrs": {
+				"uib-popover": "Index files content",
+				"popover-placement": "bottom",
+				"popover-trigger": "'mouseenter'"
+			}
+		},
 		{
 			"key": "index_cits",
 			"type": "select",

ESSArch_Core/templates/eu/EU_CSIP_AIP_v204.json

@@ -305,6 +305,23 @@
 				"popover-trigger": "'mouseenter'"
 			}
 		},
+		{
+			"key": "index_files_content",
+			"type": "select",
+			"defaultValue": true,
+			"hidden": false,
+			"templateOptions": {
+				"required": false,
+				"disabled": false,
+				"label": "Index files content",
+				"options": [{"name": "Yes","value": true},{"name": "No","value": false}]
+			},
+			"ngModelElAttrs": {
+				"uib-popover": "Index files content",
+				"popover-placement": "bottom",
+				"popover-trigger": "'mouseenter'"
+			}
+		},
 		{
 			"key": "index_cits",
 			"type": "select",
@@ -313,7 +330,7 @@
 			"templateOptions": {
 				"required": false,
 				"disabled": false,
-				"label": "Indexera CITS",
+				"label": "Index CITS",
 				"options": [{"name": "Yes","value": true},{"name": "No","value": false}]
 			},
 			"ngModelElAttrs": {

ESSArch_Core/templates/no/NO_AIP_v10.json

@@ -1006,6 +1006,23 @@
 				"popover-trigger": "'mouseenter'"
 			}
 		},
+		{
+			"key": "index_files_content",
+			"type": "select",
+			"defaultValue": true,
+			"hidden": false,
+			"templateOptions": {
+				"required": false,
+				"disabled": false,
+				"label": "Indexera filernas innehåll",
+				"options": [{"name": "Yes","value": true},{"name": "No","value": false}]
+			},
+			"ngModelElAttrs": {
+				"uib-popover": "Indexera innehåll i leveransen. Tillägg till specifikationen",
+				"popover-placement": "bottom",
+				"popover-trigger": "'mouseenter'"
+			}
+		},
 		{
 			"key": "index_cits",
 			"type": "select",

ESSArch_Core/templates/se/SE_AIP_transparent_v12.json

@@ -408,6 +408,23 @@
 				"popover-trigger": "'mouseenter'"
 			}
 		},
+		{
+			"key": "index_files_content",
+			"type": "select",
+			"defaultValue": true,
+			"hidden": false,
+			"templateOptions": {
+				"required": false,
+				"disabled": false,
+				"label": "Indexera filernas innehåll",
+				"options": [{"name": "Yes","value": true},{"name": "No","value": false}]
+			},
+			"ngModelElAttrs": {
+				"uib-popover": "Indexera innehåll i leveransen. Tillägg till specifikationen",
+				"popover-placement": "bottom",
+				"popover-trigger": "'mouseenter'"
+			}
+		},
 		{
 			"key": "index_cits",
 			"type": "select",