Merge pull request #2352 from broadinstitute/jb-duos-full-integration

Full integration for registering public studies in DUOS (SCP-6105, SCP-6106)

Author: bistline

app/lib/duos_registration_service.rb

@@ -1,5 +1,6 @@
 # service containing business logic for managing Study registrations as datasets in DUOS
 class DuosRegistrationService
+  extend Loggable
 
   # pointer to DUOS UI for auto-completing URLs
   #
@@ -14,7 +15,11 @@ def self.duos_ui_url
   # * *returns*
   #   - (DuosClient)
   def self.client
-    @client ||= DuosClient.new
+    duos_client = @client ||= DuosClient.new
+    if duos_client.access_token_expired?
+      duos_client.refresh_access_token!
+    end
+    duos_client
   end
 
   # determine if study is eligible for registering as a dataset in DUOS
@@ -80,13 +85,16 @@ def self.register_study(study)
       dataset = client.create_dataset(study)
       ids = client.identifiers_from_dataset(dataset)
       study.update(**ids)
-      Rails.logger.info "Registered #{study.accession} in DUOS as #{ids}"
+      log_message "Registered #{study.accession} in DUOS as #{ids}"
       dataset
     rescue ArgumentError => e
-      Rails.logger.error "Cannot validate #{study.accession} for DUOS: #{e.message}"
+      log_message "Cannot validate #{study.accession} for DUOS: #{e.message}", level: :error
+      nil
     rescue Faraday::Error => e
-      Rails.logger.error "Unable to register #{study.accession} in DUOS: #{e.message} (#{e.try(:response_body)})"
+      log_message "Unable to register #{study.accession} in DUOS: #{e.message} (#{e.try(:response_body)})",
+                  level: :error
       ErrorTracker.report_exception(e, client.issuer, { study: })
+      SingleCellMailer.duos_error(study, e, 'register').deliver_now
       nil
     end
   end
@@ -107,11 +115,12 @@ def self.redact_study(study)
       study.update(duos_dataset_id: nil, duos_study_id: nil)
     end
 
-    Rails.logger.info "Redacted #{study.accession} in DUOS"
+    log_message "Redacted #{study.accession} in DUOS"
     true
   rescue Faraday::Error => e
-    Rails.logger.error "Unable to redact #{study.accession} in DUOS: (#{e.message}) #{e.try(:response_body)}"
+    log_message "Unable to redact #{study.accession} in DUOS: (#{e.message}) #{e.try(:response_body)}", level: :error
     ErrorTracker.report_exception(e, client.issuer, { study: })
+    SingleCellMailer.duos_error(study, e, 'redact').deliver_now
     false
   end
 end

app/mailers/single_cell_mailer.rb

@@ -308,4 +308,16 @@ def data_retention_policy_report
       format.html { render layout: 'nightly_admin_report' }
     end
   end
+
+  def duos_error(study, error, action)
+    @study = study
+    @action = action
+    client = DuosClient.new
+    error_content = error.try(:body) || error.try(:response_body) || error.try(:reason_phrase) || error.try(:message) || error.to_s
+    @error_message = client.parse_response_body(error_content)
+    mail(to: AdminConfiguration.qa_emails,
+         subject: "[Single Cell Portal Admin Notification] DUOS #{action} error for #{@study.accession}") do |format|
+      format.html { render layout: 'nightly_admin_report' }
+    end
+  end
 end

app/models/duos_client.rb

@@ -33,7 +33,10 @@ class DuosClient
   }.freeze
 
   # identifier to append to every description
-  PLATFORM_ID = "(Platform: Single Cell Portal)".freeze
+  PLATFORM_ID = '(Platform: Single Cell Portal)'.freeze
+
+  # content for tags to use in creating data library
+  TAG_CONTENT = { tags: [PLATFORM_ID] }.freeze
 
   # initialize new client and generate access token for auth
   #
@@ -342,7 +345,8 @@ def schema_from(study)
       species: study.species_list.join(', '),
       dataCustodianEmail: study.data_custodians,
       piName: study.data_custodians.first,
-      consentGroups: [consent_values]
+      consentGroups: [consent_values],
+      data: TAG_CONTENT
     }.merge(ANVIL_VALUES).with_indifferent_access
   end
 

app/models/ingest_job.rb

@@ -501,11 +501,13 @@ def set_study_state_after_ingest
       launch_differential_expression_jobs
       launch_dot_plot_preprocess_job
       create_cell_name_indexes
+      register_duos_study
     when :ingest_expression
       set_anndata_file_info if study_file.is_anndata?
       study.delay.set_gene_count
       launch_dot_plot_preprocess_job
       launch_differential_expression_jobs
+      register_duos_study
     when :ingest_cluster
       set_cluster_point_count
       set_study_default_options
@@ -514,9 +516,11 @@ def set_study_state_after_ingest
       launch_dot_plot_preprocess_job
       launch_differential_expression_jobs
       create_cell_name_indexes
+      register_duos_study
     when :ingest_subsample
       set_subsampling_flags
       create_cell_name_indexes
+      register_duos_study
     when :differential_expression
       create_differential_expression_results
     when :ingest_differential_expression
@@ -841,6 +845,12 @@ def set_has_dot_plot_genes
     cluster_group.update(has_dot_plot_genes: true)
   end
 
+  # register eligible studies in DUOS
+  def register_duos_study
+    study.reload
+    DuosRegistrationService.register_study(study) if DuosRegistrationService.study_eligible?(study)
+  end
+
   # set appropriate flags for AnnDataFileInfo entries
   def set_anndata_file_info
     study_file.build_ann_data_file_info if study_file.ann_data_file_info.nil?

app/views/single_cell_mailer/duos_error.html.erb

@@ -0,0 +1,8 @@
+<html lang="en">
+  <body>
+    <p>The following error occurred when trying to <strong><%= @action %></strong> <%= @study.accession %> in DUOS:</p>
+    <pre>
+      <%= @error_message %>
+    </pre>
+  </body>
+</html>

db/migrate/20260224192346_register_eligible_duos_studies.rb

@@ -0,0 +1,23 @@
+class RegisterEligibleDuosStudies < Mongoid::Migration
+  def self.up
+    registered = []
+    unregistered = []
+    accessions = DuosRegistrationService.eligible_studies
+    puts "found #{accessions.count} eligible studies for DUOS registration"
+    accessions.each do |accession|
+      study = Study.find_by(accession:)
+      dataset = DuosRegistrationService.register_study(study)
+      if dataset
+        registered << accession
+      else
+        unregistered << accession
+      end
+    end
+    puts "completed!"
+    puts "registered #{registered.count} studies: #{registered.join(', ')}"
+    puts "failed to register #{unregistered.count} studies: #{unregistered.join(', ')}"
+  end
+
+  # don't reverse migration so we can re-run if necessary
+  def self.down; end
+end

test/integration/external/duos_client_test.rb

@@ -118,6 +118,8 @@ def skip_if_api_down
     participant_count = duos_data[:consentGroups].first[:numberOfParticipants]
     assert_equal 5, participant_count
     assert_equal @author.email, duos_data[:dataCustodianEmail].first
+    assert_equal DuosClient::TAG_CONTENT.with_indifferent_access,
+                 duos_data[:data].with_indifferent_access
   end
 
   test 'should load dataset JSON schema from DUOS' do