Sensitive Materials Feature Branch (#1540)

K8Sewell · jpengst · JP Engstrom · web-flow · commit aef305d4ea68 · 2025-07-21T13:15:01.000-07:00
* Sensitive Materials Dropdown (#1538) --------- Co-authored-by: JP Engstrom <jpengstrom@macbook-pro.lan> * Sensitive Materials to Templates (#1541) Co-authored-by: JP Engstrom <jpengstrom@macbook-pro.lan> * added sensitive_materials to params (#1546) * added sensitive_materials to params * rubocop --------- Co-authored-by: JP Engstrom <jpengstrom@MWPF4KM52A.yu.yale.edu> * Sensitive Materials Export BP (#1547) Co-authored-by: JP Engstrom <jpengstrom@MWPF4KM52A.yu.yale.edu> * adds sensitive_materials to create and update BP (#1542) Co-authored-by: JP Engstrom <jpengstrom@macbook-pro.lan> * adds sensitive_materials to parent object datatable (#1548) Co-authored-by: JP Engstrom <jpengstrom@MWPF4KM52A.yu.yale.edu> * defaults sensitive materials dropdown to nil (#1550) Co-authored-by: JP Engstrom <jpengstrom@macbook-pro.lan> * remove pre-selected option (#1552) Co-authored-by: JP Engstrom <jpengstrom@macbook-pro.lan> * Placeholder thumbnail (#1551) Co-authored-by: JP Engstrom <jpengstrom@macbook-pro.lan> Co-authored-by: JP Engstrom <jpengstrom@MWPF51CVBT.yu.yale.edu> * Fix spec --------- Co-authored-by: JP Engstrom <jpengst@gmail.com> Co-authored-by: JP Engstrom <jpengstrom@macbook-pro.lan> Co-authored-by: JP Engstrom <jpengstrom@MWPF4KM52A.yu.yale.edu> Co-authored-by: JP Engstrom <jpengstrom@MWPF51CVBT.yu.yale.edu>
diff --git a/app/controllers/parent_objects_controller.rb b/app/controllers/parent_objects_controller.rb
@@ -293,7 +293,8 @@ def parent_object_params
                                                        :viewing_direction,
                                                        :permission_set_id,
                                                        :display_layout, :representative_child_oid, :rights_statement, :extent_of_digitization,
-                                                       :digitization_note, :digitization_funding_source, :redirect_to, :preservica_uri, :digital_object_source, :preservica_representation_type,
+                                                       :digitization_note, :digitization_funding_source, :redirect_to, :preservica_uri, :digital_object_source,
+                                                       :preservica_representation_type, :sensitive_materials,
                                                        :mms_id, :alma_holding, :alma_item)
     cur_params[:admin_set] = AdminSet.find_by(key: cur_params[:admin_set]) if cur_params[:admin_set]
     cur_params
diff --git a/app/datatables/parent_object_datatable.rb b/app/datatables/parent_object_datatable.rb
@@ -42,6 +42,7 @@ def view_columns
       last_id_update: { source: "ParentObject.last_id_update", orderable: true },
       visibility: { source: "ParentObject.visibility", cond: :string_eq, searchable: true, options: ["Public", "Yale Community Only", "Private", "Open with Permission"], orderable: true },
       permission_set: { source: "OpenWithPermission::PermissionSet.label", cond: :string_eq, options: @set_labels, searchable: true, orderable: true },
+      sensitive_materials: { source: "ParentObject.sensitive_materials", cond: :string_eq, searchable: true, options: ["Yes", "No"], orderable: true },
       extent_of_digitization: { source: "ParentObject.extent_of_digitization", cond: :string_eq, searchable: true, options: ["Completely digitized", "Partially digitized"], orderable: true },
       digitization_note: { source: "ParentObject.digitization_note", cond: :like, searchable: true, orderable: true },
       digitization_funding_source: { source: "ParentObject.digitization_funding_source", cond: :like, searchable: true, orderable: true },
@@ -81,6 +82,7 @@ def data
         last_id_update: parent_object.last_id_update,
         visibility: parent_object.visibility,
         permission_set: parent_object&.permission_set&.label,
+        sensitive_materials: parent_object&.sensitive_materials,
         extent_of_digitization: parent_object.extent_of_digitization,
         digitization_note: parent_object.digitization_note,
         digitization_funding_source: parent_object.digitization_funding_source,
diff --git a/app/models/concerns/create_parent_object.rb b/app/models/concerns/create_parent_object.rb
@@ -71,6 +71,7 @@ def create_new_parent_csv
         parent_object.digitization_note = row['digitization_note']
         parent_object.digitization_funding_source = row['digitization_funding_source']
         parent_object.rights_statement = row['rights_statement']
+        parent_object.sensitive_materials = row['sensitive_materials']
 
         if row['visibility'] == 'Open with Permission'
           permission_set = OpenWithPermission::PermissionSet.find_by(key: row['permission_set_key'])
diff --git a/app/models/concerns/csv_exportable.rb b/app/models/concerns/csv_exportable.rb
@@ -12,7 +12,7 @@ def parent_headers
      'container_grouping', 'mms_id', 'alma_holding', 'alma_item', 'bib', 'holding', 'item', 'barcode', 'aspace_uri',
      'digital_object_source', 'preservica_uri', 'preservica_representation_type', 'last_ladybird_update',
      'last_voyager_update', 'last_sierra_update', 'last_aspace_update', 'last_id_update', 'visibility', 'permission_set_key',
-     'extent_of_digitization', 'digitization_note', 'digitization_funding_source', 'rights_statement', 'project_identifier', 'full_text']
+     'sensitive_materials', 'extent_of_digitization', 'digitization_note', 'digitization_funding_source', 'rights_statement', 'project_identifier', 'full_text']
   end
 
   # rubocop:disable Metrics/AbcSize
@@ -30,7 +30,7 @@ def parent_output_csv(*admin_set_id)
                   po.child_object_count, po.authoritative_json&.[]('title')&.first, po.call_number, po.container_grouping, po.mms_id, po.alma_holding, po.alma_item, po.bib, po.holding, po.item,
                   po.barcode, po.aspace_uri, po.digital_object_source, po.preservica_uri, po.preservica_representation_type,
                   po.last_ladybird_update, po.last_voyager_update, po.last_sierra_update,
-                  po.last_aspace_update, po.last_id_update, po.visibility, po&.permission_set&.key, po.extent_of_digitization,
+                  po.last_aspace_update, po.last_id_update, po.visibility, po&.permission_set&.key, po&.sensitive_materials, po.extent_of_digitization,
                   po.digitization_note, po.digitization_funding_source, po.rights_statement, po.project_identifier, extent_of_full_text(po)]
         else
           csv << [po[:id], po[:row2], '-', po[:csv_message], '', '']
@@ -134,7 +134,7 @@ def export_parent_metadata
              po.mms_id, po.alma_holding, po.alma_item, po.bib, po.holding, po.item,
              po.barcode, po.aspace_uri, po.digital_object_source, po.preservica_uri, po.preservica_representation_type,
              po.last_ladybird_update, po.last_voyager_update, po.last_sierra_update,
-             po.last_aspace_update, po.last_id_update, po.visibility, po&.permission_set&.key, po.extent_of_digitization,
+             po.last_aspace_update, po.last_id_update, po.visibility, po&.permission_set&.key, po.sensitive_materials, po.extent_of_digitization,
              po.digitization_note, po.digitization_funding_source, po.rights_statement, po.project_identifier, extent_of_full_text(po)]
       csv_rows << row
     end
diff --git a/app/models/concerns/solr_indexable.rb b/app/models/concerns/solr_indexable.rb
@@ -195,6 +195,7 @@ def to_solr(json_to_index = nil)
         rights_ssim: rights_statement,
         rights_tesim: rights_statement,
         scale_tesim: json_to_index["scale"],
+        sensitive_materials_ssi: sensitive_materials.presence,
         series_ssi: json_to_index["series"],
         series_sort_ssi: series_sort(json_to_index),
         source_ssim: json_to_index["source"], # refers to source of metadata, e.g. Ladybird, Voyager, etc.
diff --git a/app/models/concerns/updatable.rb b/app/models/concerns/updatable.rb
@@ -189,7 +189,7 @@ def remove_blanks(row, parent_object)
 
   def validate_field(parent_object, row)
     fields = ['aspace_uri', 'barcode', 'bib', 'digital_object_source', 'digitization_funding_source', 'digitization_note', 'holding', 'item', 'mms_id', 'alma_holding', 'alma_item',
-              'preservica_representation_type', 'preservica_uri', 'project_identifier', 'rights_statement', 'redirect_to']
+              'preservica_representation_type', 'preservica_uri', 'project_identifier', 'rights_statement', 'redirect_to', 'sensitive_materials']
     validation_fields = { "display_layout" => 'viewing_hints', "extent_of_digitization" => 'extent_of_digitizations', "viewing_direction" => 'viewing_directions', "visibility" => 'visibilities' }
     row, blanks = remove_blanks(row, parent_object)
     processed_fields = {}
diff --git a/app/models/parent_object.rb b/app/models/parent_object.rb
@@ -42,6 +42,7 @@ class ParentObject < ApplicationRecord # rubocop:disable Metrics/ClassLength
   validates :preservica_representation_type, format: { with: /\A(Preservation|preservation|Access)/, message: "can't be None when Digital Object Source is Preservica" }, if: proc { digital_object_source == "Preservica" || digital_object_source == "preservica" }
   # rubocop:enable Layout/LineLength
   validate :validate_visibility
+  validate :validate_sensitive_materials
   before_save :check_for_redirect
   before_save :check_permission_set
 
@@ -772,4 +773,9 @@ def should_index?
   def should_create_manifest_and_pdf?
     !redirect_to.present?
   end
+
+  def validate_sensitive_materials
+    return if sensitive_materials.blank? || sensitive_materials == 'Yes' || sensitive_materials == 'No'
+    errors.add(:sensitive_materials, "must be 'Yes' or 'No'")
+  end
 end
diff --git a/app/views/parent_objects/_form.html.erb b/app/views/parent_objects/_form.html.erb
@@ -103,6 +103,14 @@
   </div>
   <br>
 
+  <div class="form-row">
+    <div class="col-med-9">
+      <%= form.label :sensitive_materials, "Sensitive Materials" %>
+      <%= form.select :sensitive_materials, [[nil], ['Yes'], ['No']] %>
+    </div>
+  </div>
+  <br>
+
   <div class="form-row">
     <div class="col-med-9">
       <%= form.label :rights_statement %>
diff --git a/app/views/parent_objects/index.html.erb b/app/views/parent_objects/index.html.erb
@@ -45,6 +45,7 @@
           <th scope='col'>Last ID Update</th>
           <th scope='col'>Visibility</th>
           <th scope='col'>Permission Set Label</th>
+          <th scope='col'>Sensitive Materials</th>
           <th scope='col'>Extent of Digitization</th>
           <th scope='col'>Digitization Note</th>
           <th scope='col'>Digitization Funding Source</th>
diff --git a/app/views/parent_objects/show.html.erb b/app/views/parent_objects/show.html.erb
@@ -93,6 +93,10 @@
         <td class="key">Permission Set:</td>
         <td><%= @parent_object&.permission_set&.label %></td>
       </tr>
+      <tr class='table-row'>
+        <td class="key">Sensitive Materials:</td>
+        <td><%= @parent_object&.sensitive_materials %></td>
+      </tr>
       <tr class='table-row'>
         <td class="key">Rights Statement:</td>
         <td><%= sanitize @parent_object.rights_statement, tags: %w(a), attributes: %w(href) %></td>
diff --git a/db/migrate/20250528222739_add_sensitive_materials_to_parent_object.rb b/db/migrate/20250528222739_add_sensitive_materials_to_parent_object.rb
@@ -0,0 +1,5 @@
+class AddSensitiveMaterialsToParentObject < ActiveRecord::Migration[7.0]
+  def change
+    add_column :parent_objects, :sensitive_materials, :string
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
@@ -10,7 +10,9 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.0].define(version: 2025_05_22_205537) do
+
+ActiveRecord::Schema[7.0].define(version: 2025_05_28_222739) do
+
   # These are extensions that must be enabled in order to support this database
   enable_extension "pgcrypto"
   enable_extension "plpgsql"
@@ -280,6 +282,7 @@
     t.text "extent_of_full_text", default: "None"
     t.jsonb "sierra_json"
     t.datetime "last_sierra_update", precision: nil
+    t.string "sensitive_materials"
     t.jsonb "alma_json"
     t.datetime "last_alma_update"
     t.string "mms_id"
diff --git a/public/batch_processes/templates/create_parent_objects.csv b/public/batch_processes/templates/create_parent_objects.csv
@@ -1 +1 @@
-oid,admin_set,source,aspace_uri,mms_id,alma_holding,alma_item,bib,holding,item,barcode,visibility,permission_set_key,digital_object_source,preservica_uri,preservica_representation_type,extent_of_digitization,digitization_note,digitization_funding_source,rights_statement,viewing_direction,display_layout
+oid,admin_set,source,aspace_uri,mms_id,alma_holding,alma_item,bib,holding,item,barcode,visibility,permission_set_key,digital_object_source,preservica_uri,preservica_representation_type,extent_of_digitization,digitization_note,digitization_funding_source,sensitive_materials,rights_statement,viewing_direction,display_layout
diff --git a/public/batch_processes/templates/update_parent_objects.csv b/public/batch_processes/templates/update_parent_objects.csv
@@ -1 +1 @@
-oid,source,admin_set,project_identifier,visibility,permission_set_key,rights_statement,extent_of_digitization,digitization_note,digitization_funding_source,mms_id,alma_holding,alma_item,bib,holding,item,barcode,aspace_uri,viewing_direction,display_layout,redirect_to,digital_object_source,preservica_uri,preservica_representation_type
+oid,source,admin_set,project_identifier,visibility,permission_set_key,rights_statement,extent_of_digitization,digitization_note,digitization_funding_source,mms_id,alma_holding,alma_item,bib,holding,item,barcode,aspace_uri,viewing_direction,display_layout,redirect_to,sensitive_materials,digital_object_source,preservica_uri,preservica_representation_type
diff --git a/spec/datatables/parent_object_datatable_spec.rb b/spec/datatables/parent_object_datatable_spec.rb
@@ -24,6 +24,7 @@
                            alma_item: "34",
                            project_identifier: '67',
                            digital_object_source: "Preservica",
+                           sensitive_materials: "Yes",
                            preservica_representation_type: "Access",
                            preservica_uri: "/preservica_uri")
     output = ParentObjectDatatable.new(datatable_sample_params(columns), view_context: parent_object_datatable_view_mock, current_ability: Ability.new(user)).data
@@ -47,6 +48,7 @@
       preservica_uri: "/preservica_uri",
       digitization_note: nil,
       digitization_funding_source: nil,
+      sensitive_materials: "Yes",
       extent_of_digitization: nil,
       holding: nil,
       item: nil,
diff --git a/spec/fixtures/csv/aspace_parent.csv b/spec/fixtures/csv/aspace_parent.csv
@@ -1,2 +1,2 @@
-oid,admin_set,source,aspace_uri,bib,holding,item,barcode,visibility,extent_of_digitization
-,brbl,aspace,/repositories/12/archival_objects/781086,4320085,,,39002075038423,Public,Partially digitized
+oid,admin_set,source,aspace_uri,bib,holding,item,barcode,visibility,extent_of_digitization,sensitive_materials
+,brbl,aspace,/repositories/12/archival_objects/781086,4320085,,,39002075038423,Public,Partially digitized,Yes
diff --git a/spec/fixtures/csv/create_invalid_sensitive_materials_parent.csv b/spec/fixtures/csv/create_invalid_sensitive_materials_parent.csv
@@ -0,0 +1,2 @@
+oid,admin_set,source,aspace_uri,bib,holding,item,barcode,visibility,extent_of_digitization,sensitive_materials
+,brbl,aspace,/repositories/12/archival_objects/781086,4320085,,,39002075038423,Public,Partially digitized,Yess
diff --git a/spec/fixtures/csv/update_example_owp.csv b/spec/fixtures/csv/update_example_owp.csv
@@ -1,2 +1,2 @@
-oid,source,admin_set,project_identifier,visibility,permission_set_key,rights_statement,extent_of_digitization,digitization_note,bib,holding,item,barcode,aspace_uri,viewing_direction,display_layout,digital_object_source,preservica_uri,preservica_representation_type
-2034600,ladybird,brbl,Beinecke,Open with Permission,PS Key,The use of this image may be subject to the copyright law of the United States,Completely digitized,5678,12307100,temporary,reel,39002102340669,/repositories/11/archival_objects/515305,left-to-right,paged,,,
+oid,source,admin_set,project_identifier,visibility,permission_set_key,rights_statement,extent_of_digitization,digitization_note,bib,holding,item,barcode,aspace_uri,viewing_direction,display_layout,sensitive_materials,digital_object_source,preservica_uri,preservica_representation_type
+2034600,ladybird,brbl,Beinecke,Open with Permission,PS Key,The use of this image may be subject to the copyright law of the United States,Completely digitized,5678,12307100,temporary,reel,39002102340669,/repositories/11/archival_objects/515305,left-to-right,paged,Yes,,,
diff --git a/spec/models/batch_process_create_parent_spec.rb b/spec/models/batch_process_create_parent_spec.rb
@@ -8,6 +8,7 @@
   let(:admin_set_one) { FactoryBot.create(:admin_set, key: 'jss') }
   let(:create_owp_parent) { Rack::Test::UploadedFile.new(Rails.root.join(fixture_path, "csv", "create_owp_parent.csv")) }
   let(:create_invalid_owp_parent) { Rack::Test::UploadedFile.new(Rails.root.join(fixture_path, "csv", "create_invalid_owp_parent.csv")) }
+  let(:create_invalid_sensitive_materials_parent) { Rack::Test::UploadedFile.new(Rails.root.join(fixture_path, "csv", "create_invalid_sensitive_materials_parent.csv")) }
   let(:mini_create_owp_parent) { Rack::Test::UploadedFile.new(Rails.root.join(fixture_path, "csv", "mini_owp_parent.csv")) }
   let(:permission_set) { FactoryBot.create(:permission_set, key: "PS Key") }
   let(:no_oid_parent) { Rack::Test::UploadedFile.new(Rails.root.join(fixture_path, 'csv', 'parent_no_oid.csv')) }
@@ -54,6 +55,7 @@
         po = ParentObject.first
         expect(po.bib).to eq('4320085')
         expect(po.aspace_uri).to eq('/repositories/12/archival_objects/781086')
+        expect(po.sensitive_materials).to eq('Yes')
       end
       it 'can create a parent object from alma' do
         expect do
@@ -74,6 +76,14 @@
         po = ParentObject.first
         expect(po.oid).not_to be_nil
       end
+      it 'fails creating a parent_object with invalid Sensitive Materials value' do
+        permission_set.add_administrator(user)
+        expect do
+          batch_process.file = create_invalid_sensitive_materials_parent
+          batch_process.save
+        end.not_to change { ParentObject.count }
+        expect(batch_process.batch_ingest_events[0].reason).to eq("Skipping row [2] Unable to save parent: Validation failed: Sensitive materials must be 'Yes' or 'No'.")
+      end
       it 'can create an OwP parent_object' do
         permission_set.add_administrator(user)
         expect do
diff --git a/spec/models/batch_process_update_parent_spec.rb b/spec/models/batch_process_update_parent_spec.rb
@@ -141,6 +141,7 @@
       expect(po_updated.viewing_direction).to eq "left-to-right"
       expect(po_updated.visibility).to eq "Open with Permission"
       expect(po_updated.permission_set_id).to eq permission_set.id
+      expect(po_updated.sensitive_materials).to eq "Yes"
     end
 
     it "does not update successfully if permission set is invalid" do
diff --git a/spec/requests/batch_processes_request_spec.rb b/spec/requests/batch_processes_request_spec.rb
@@ -173,7 +173,8 @@
       get download_template_batch_processes_url(batch_action: "create parent objects")
       expect(response).to have_http_status(:success)
       expect(response.content_type).to eq("text/csv; charset=utf-8")
-      expect(response.body.to_s).to eq("\xEF\xBB\xBFoid,admin_set,source,aspace_uri,mms_id,alma_holding,alma_item,bib,holding,item,barcode,visibility,permission_set_key,digital_object_source,preservica_uri,preservica_representation_type,extent_of_digitization,digitization_note,digitization_funding_source,rights_statement,viewing_direction,display_layout")
+      # byebug
+      expect(response.body.to_s).to eq("\xEF\xBB\xBFoid,admin_set,source,aspace_uri,mms_id,alma_holding,alma_item,bib,holding,item,barcode,visibility,permission_set_key,digital_object_source,preservica_uri,preservica_representation_type,extent_of_digitization,digitization_note,digitization_funding_source,sensitive_materials,rights_statement,viewing_direction,display_layout\n")
     end
     # rubocop:enable Layout/LineLength
 
diff --git a/spec/system/batch_process_spec.rb b/spec/system/batch_process_spec.rb
@@ -168,7 +168,7 @@
       let(:brbl) { AdminSet.find_by_key("brbl") }
       let(:other_admin_set) { FactoryBot.create(:admin_set) }
       # rubocop:disable Layout/LineLength
-      let(:parent_object) { FactoryBot.create(:parent_object, oid: 2_034_600, admin_set: brbl, digital_object_source: "Preservica", preservica_uri: "/preservica_uri", preservica_representation_type: "Access") }
+      let(:parent_object) { FactoryBot.create(:parent_object, oid: 2_034_600, admin_set: brbl, digital_object_source: "Preservica", preservica_uri: "/preservica_uri", preservica_representation_type: "Access", sensitive_materials: "Yes") }
       let(:parent_object2) { FactoryBot.create(:parent_object, oid: 2_005_512, admin_set: other_admin_set) }
       let(:parent_object3) { FactoryBot.create(:parent_object, oid: 2_004_548, admin_set: brbl) }
       let(:user) { FactoryBot.create(:user) }
@@ -201,6 +201,7 @@
         expect(BatchProcess.last.export_parent_metadata).to include "brbl"
         expect(BatchProcess.last.export_parent_metadata).to include "full_text"
         expect(BatchProcess.last.export_parent_metadata).to include "last_sierra_update"
+        expect(BatchProcess.last.export_parent_metadata).to include "Yes"
       end
 
       it "uploads a CSV of parent objects in order to create export of parent objects metadata fails 2005512 row due to admin set permissions" do
@@ -231,6 +232,7 @@
         expect(BatchProcess.last.parent_output_csv).to include "brbl"
         expect(BatchProcess.last.parent_output_csv).to include "full_text"
         expect(BatchProcess.last.parent_output_csv).to include "last_sierra_update"
+        expect(BatchProcess.last.parent_output_csv).to include "Yes"
         expect(BatchProcess.last.parent_output_csv).not_to include "2005512"
       end
 
