Merge pull request #2232 from broadinstitute/jb-raw-slot-ux

Adding `raw_location` for AnnData files (SCP-5961)

Author: bistline

app/controllers/api/v1/study_files_controller.rb

@@ -749,7 +749,7 @@ def study_file_params
           ],
           expression_file_info_attributes: [
             :_id, :_destroy, :library_preparation_protocol, :units, :biosample_input_type, :modality, :is_raw_counts,
-            raw_counts_associations: []
+            :raw_location, raw_counts_associations: []
           ],
           heatmap_file_info_attributes: [:id, :_destroy, :custom_scaling, :color_min, :color_max, :legend_label],
           cluster_form_info_attributes: [
@@ -758,8 +758,8 @@ def study_file_params
             :external_link_description, spatial_cluster_associations: []
           ],
           metadata_form_info_attributes: [:_id, :use_metadata_convention, :description],
-          extra_expression_form_info_attributes: [:_id, :taxon_id, :description, :y_axis_label],
-          ann_data_file_info_attributes: [:_id, :reference_file, :data_fragments],
+          extra_expression_form_info_attributes: [:_id, :taxon_id, :description, :y_axis_label, :raw_location],
+          ann_data_file_info_attributes: [:_id, :reference_file, :data_fragments, :raw_location],
           differential_expression_file_info_attributes: [
             :_id, :clustering_association, :annotation_name, :annotation_scope, :computational_method,
             :gene_header, :group_header, :comparison_group_header,

app/javascript/components/upload/AnnDataExpressionStep.jsx

@@ -10,6 +10,7 @@ const DEFAULT_NEW_PROCESSED_FILE = {
     is_raw_counts: false,
     biosample_input_type: 'Whole cell',
     modality: 'Transcriptomic: unbiased',
+    raw_location: '',
     raw_counts_associations: []
   },
   file_type: 'Expression Matrix'

app/javascript/components/upload/ExpressionFileForm.jsx

@@ -8,6 +8,9 @@ import ExpandableFileForm from './ExpandableFileForm'
 
 import { TextFormField } from './form-components'
 import { findBundleChildren, validateFile } from './upload-utils'
+import { faQuestionCircle } from '@fortawesome/free-solid-svg-icons'
+import { OverlayTrigger, Popover } from 'react-bootstrap'
+import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'
 
 const REQUIRED_FIELDS = [{ label: 'species', propertyName: 'taxon_id' },
   { label: 'Biosample input type', propertyName: 'expression_file_info.biosample_input_type' },
@@ -19,6 +22,9 @@ const RAW_COUNTS_REQUIRED_FIELDS = REQUIRED_FIELDS.concat([{
 const PROCESSED_ASSOCIATION_FIELD = [
   { label: 'Associated raw counts files', propertyName: 'expression_file_info.raw_counts_associations' }
 ]
+const RAW_LOCATION_FIELD = [
+  { label: 'Raw count data location', propertyName: 'raw_location' },
+]
 
 /** renders a form for editing/uploading an expression file (raw or processed) and any bundle children */
 export default function ExpressionFileForm({
@@ -49,6 +55,10 @@ export default function ExpressionFileForm({
   if (rawCountsRequired && !isRawCountsFile ) {
     requiredFields = requiredFields.concat(PROCESSED_ASSOCIATION_FIELD)
   }
+  const requireLocation = (rawCountsRequired || isRawCountsFile) && isAnnDataExperience
+  if (requireLocation) {
+    requiredFields = requiredFields.concat(RAW_LOCATION_FIELD)
+  }
   const validationMessages = validateFile({ file, allFiles, allowedFileExts, requiredFields, isAnnDataExperience })
 
   const associatedRawCounts = !isAnnDataExperience && file.expression_file_info.raw_counts_associations.map(id => ({
@@ -61,9 +71,45 @@ export default function ExpressionFileForm({
     setShowRawCountsUnits(rawCountsVal)
   }
 
+  /** create the tooltip and message for the .obsm key name section */
+  function rawSlotMessage() {
+    const rawSlotToolTip = <span>
+      <OverlayTrigger
+        trigger={['hover', 'focus']}
+        rootClose placement="top"
+        delayHide={1500}
+        overlay={rawSlotHelpContent()}>
+        <span> Raw count data location * <FontAwesomeIcon icon={faQuestionCircle}/></span>
+      </OverlayTrigger>
+    </span>
+
+    return <span >
+      {rawSlotToolTip}
+    </span>
+  }
+
+  /** gets the popup message to describe .obsm keys */
+  function rawSlotHelpContent() {
+    const layersLink = <a href="https://anndata.readthedocs.io/en/latest/generated/anndata.AnnData.layers.html"
+                          target="_blank">layers</a>
+    const rawLink = <a href="https://anndata.readthedocs.io/en/latest/generated/anndata.AnnData.raw.html"
+                                target="_blank">.raw</a>
+    return <Popover id="cluster-obsm-key-name-popover" className="tooltip-wide">
+      <div>
+        Location of raw count data in your AnnData file. This can be the raw slot ({rawLink}) or the name of a layer in
+        the {layersLink} section.
+      </div>
+    </Popover>
+  }
+
   return <ExpandableFileForm {...{
-    file, allFiles, updateFile, saveFile,
-    allowedFileExts, deleteFile, validationMessages, bucketName, isInitiallyExpanded, isAnnDataExperience
+    file,
+    allFiles,
+    updateFile,
+    saveFile,
+    allowedFileExts,
+    deleteFile,
+    validationMessages, bucketName, isInitiallyExpanded, isAnnDataExperience
   }}>
     {!isAnnDataExperience &&
     <div className="form-group">
@@ -123,25 +169,32 @@ export default function ExpressionFileForm({
     { isAnnDataExperience &&
       <div className="row">
         <div className="form-radio col-sm-4">
-          <label className="labeled-select">I have raw count data in the <strong>adata.raw</strong> slot</label>
+          <label className="labeled-select">I have raw count data</label>
           <label className="sublabel">
             <input type="radio"
                    name={`anndata-raw-counts-${file._id}`}
                    value="true"
                    checked={isRawCountsFile}
-                   onChange={e => toggleIsRawCounts(true) } />
+                   onChange={e => toggleIsRawCounts(true)}/>
             &nbsp;Yes
           </label>
           <label className="sublabel">
             <input type="radio"
                    name={`anndata-raw-counts-${file._id}`}
                    value="false"
                    checked={!isRawCountsFile}
-                   onChange={e => toggleIsRawCounts(false) }/>
+                   onChange={e => toggleIsRawCounts(false)}/>
             &nbsp;No
           </label>
         </div>
-        {showRawCountsUnits && <div className="col-sm-8">
+        {requireLocation && <div className="col-sm-4">
+          <TextFormField label={rawSlotMessage()}
+                         fieldName="raw_location"
+                         file={file}
+                         updateFile={updateFile}
+                         placeholderText='Specify .raw or name of layer'/></div>
+        }
+        { showRawCountsUnits && <div className="col-sm-4">
           <ExpressionFileInfoSelect label="Units *"
                                     propertyName="units"
                                     rawOptions={fileMenuOptions.units}

app/lib/differential_expression_service.rb

@@ -161,6 +161,7 @@ def self.run_differential_expression_job(cluster_group, study, user, annotation_
     elsif raw_matrix.file_type == 'AnnData'
       de_params[:matrix_file_type] = 'h5ad'
       de_params[:file_size] = raw_matrix.upload_file_size
+      de_params[:raw_location] = raw_matrix.ann_data_file_info.raw_location
     else
       de_params[:matrix_file_type] = 'dense'
     end

app/lib/file_parse_service.rb

@@ -112,13 +112,15 @@ def self.run_parse_job(study_file, study, user, reparse: false, persist_on_fail:
             )
           elsif study_file.needs_raw_counts_extraction?
             params_object = AnnDataIngestParameters.new(
-              anndata_file: study_file.gs_url, extract: %w[raw_counts], obsm_keys: nil,
+              anndata_file: study_file.gs_url, extract: %w[raw_counts],
+              raw_location: study_file.ann_data_file_info.raw_location, obsm_keys: nil,
               file_size: study_file.upload_file_size
             )
           else
             params_object = AnnDataIngestParameters.new(
               anndata_file: study_file.gs_url, obsm_keys: study_file.ann_data_file_info.obsm_key_names,
-              file_size: study_file.upload_file_size, extract_raw_counts: study_file.is_raw_counts_file?
+              file_size: study_file.upload_file_size, extract_raw_counts: study_file.is_raw_counts_file?,
+              raw_location: study_file.ann_data_file_info.raw_location
             )
           end
           # TODO extract and parse Raw Exp Data (SCP-4710)

app/models/ann_data_file_info.rb

@@ -18,7 +18,7 @@ class AnnDataFileInfo
       y_axis_max z_axis_min z_axis_max external_link_url external_link_title external_link_description
       parse_status spatial_cluster_associations
     ],
-    expression: %i[_id data_type taxon_id description expression_file_info y_axis_label]
+    expression: %i[_id data_type taxon_id description expression_file_info y_axis_label raw_location]
   }.freeze
 
   # required keys for data_fragments, by type
@@ -32,6 +32,8 @@ class AnnDataFileInfo
   field :has_expression, type: Boolean, default: false
   # controls whether or not to ingest data (true: should not ingest data, this is like an 'Other' file)
   field :reference_file, type: Boolean, default: true
+  # location of raw count data, either .raw attribute or in layers[{name}]
+  field :raw_location, type: String, default: ''
   # information from form about data contained inside AnnData file, such as names/descriptions
   # examples:
   # {
@@ -40,8 +42,8 @@ class AnnDataFileInfo
   # }
   # { _id: '6033f531e241391884633748', data_type: :expression, description: 'log(TMP) expression' }
   field :data_fragments, type: Array, default: []
-  before_validation :set_default_cluster_fragments!, :sanitize_fragments!
-  validate :validate_fragments
+  before_validation :set_default_cluster_fragments!, :set_raw_location!, :sanitize_fragments!
+  validate :validate_fragments, :enforce_raw_location
   after_validation :update_expression_file_info
 
   # collect data frame key_names for clustering data inside AnnData flle
@@ -82,6 +84,8 @@ def merge_form_data(form_data)
         fragments << extract_form_fragment(fragment_form, key, *allowed_params)
       when :expression
         merged_data[:taxon_id] = fragment_form[:taxon_id]
+        anndata_info_attributes[:raw_location] = merged_data.dig(:expression_file_info_attributes, :raw_location)
+        merged_data[:expression_file_info_attributes]&.delete(:raw_location) # prevent UnknownAttribute error
         merged_exp_fragment = fragment_form.merge(expression_file_info: merged_data[:expression_file_info_attributes])
         fragments << extract_form_fragment(merged_exp_fragment, key, *allowed_params)
       end
@@ -151,9 +155,18 @@ def update_expression_file_info
     return nil if reference_file || exp_fragment.nil? || exp_info.nil?
 
     info_update = exp_fragment.with_indifferent_access[:expression_file_info]
+    info_update.delete(:raw_location) if info_update[:raw_location]
     exp_info.assign_attributes(**info_update) if info_update
   end
 
+  # pull out raw_location from expression fragment and set as top-level attribute for ease of access
+  def set_raw_location!
+    exp_fragment = find_fragment(data_type: :expression) || fragments_by_type(:expression).first
+    return nil if reference_file || exp_fragment.nil?
+
+    self.raw_location = exp_fragment.with_indifferent_access[:raw_location]
+  end
+
   # extract description field from expression fragment to use as axis label
   def expression_axis_label
     exp_fragment = find_fragment(data_type: :expression) || fragments_by_type(:expression).first
@@ -218,7 +231,7 @@ def validate_fragments
     REQUIRED_FRAGMENT_KEYS.each do |data_type, keys|
       fragments = fragments_by_type(data_type)
       fragments.each do |fragment|
-        unset_units_in_exp_fragment(fragment) if data_type == :expression
+        unset_fields_in_exp_fragment(fragment) if data_type == :expression
         missing_keys = keys.map(&:to_s) - fragment.keys.map(&:to_s)
         missing_values = keys.select { |key| fragment[key].blank? }
         next if missing_keys.empty? && missing_values.empty?
@@ -239,16 +252,23 @@ def validate_fragments
     end
   end
 
-  # unset units in expression fragment since form data won't have value
+  def enforce_raw_location
+    if study_file.is_raw_counts_file? && !reference_file && raw_location.blank?
+      errors.add(:raw_location, 'must have a value for raw count matrices')
+    end
+  end
+
+  # unset units and raw_location in expression fragment since form data won't have value
   # element must be replaced by index in order to persist
-  def unset_units_in_exp_fragment(fragment)
+  def unset_fields_in_exp_fragment(fragment)
     exp_info = fragment[:expression_file_info]
     return nil unless exp_info
 
     unless exp_info[:is_raw_counts]
       exp_info.delete(:units)
       frag_idx = fragment_index_of(fragment)
       data_fragments[frag_idx][:expression_file_info] = exp_info
+      data_fragments[frag_idx][:raw_location] = ''
     end
   end
 end

app/models/ann_data_ingest_parameters.rb

@@ -39,6 +39,7 @@ class AnnDataIngestParameters
     domain_ranges: nil,
     extract: %w[cluster metadata processed_expression],
     extract_raw_counts: false,
+    raw_location: nil,
     cell_metadata_file: nil,
     ingest_cell_metadata: false,
     study_accession: nil,
@@ -60,6 +61,7 @@ class AnnDataIngestParameters
             format: { with: Parameterizable::GS_URL_REGEXP, message: 'is not a valid GS url' },
             allow_blank: true
   validates :machine_type, inclusion: Parameterizable::GCE_MACHINE_TYPES
+  validates :raw_location, presence: true, if: proc { extract_raw_counts || extract.to_a.include?(:raw_counts) }
 
   def initialize(attributes = nil)
     super

app/models/differential_expression_parameters.rb

@@ -25,6 +25,7 @@ class DifferentialExpressionParameters
   # matrix_file_path: raw counts matrix with source expression data
   # matrix_file_type: type of raw counts matrix (dense, sparse)
   # matrix_file_id: BSON ID of raw matrix StudyFile for associations
+  # raw_location: slot in AnnData file where raw data resides (only needed for AnnData matrices)
   # gene_file (optional): genes/features file for sparse matrix
   # barcode_file (optional): barcodes file for sparse matrix
   # machine_type (optional): override for default ingest machine type (uses 'n2d-highmem-8')
@@ -43,6 +44,7 @@ class DifferentialExpressionParameters
     matrix_file_path: nil,
     matrix_file_type: nil,
     matrix_file_id: nil,
+    raw_location: nil,
     gene_file: nil,
     barcode_file: nil,
     machine_type: 'n2d-highmem-8',
@@ -62,6 +64,7 @@ class DifferentialExpressionParameters
   validates :de_type, inclusion: %w[rest pairwise]
   validates :group1, :group2, presence: true, if: -> { de_type == 'pairwise' }
   validates :matrix_file_type, inclusion: %w[dense mtx h5ad]
+  validates :raw_location, presence: true, if: -> { matrix_file_type == 'h5ad' }
   validates :machine_type, inclusion: Parameterizable::GCE_MACHINE_TYPES
   validates :gene_file, :barcode_file,
             presence: true,

config/application.rb

@@ -29,7 +29,7 @@ class Application < Rails::Application
     config.middleware.use Rack::Brotli
 
     # Docker image for file parsing via scp-ingest-pipeline
-    config.ingest_docker_image = 'gcr.io/broad-singlecellportal-staging/scp-ingest-pipeline:1.40.1'
+    config.ingest_docker_image = 'gcr.io/broad-singlecellportal-staging/scp-ingest-pipeline:1.41.1'
 
     # Docker image for image pipeline jobs
     config.image_pipeline_docker_image = 'gcr.io/broad-singlecellportal-staging/image-pipeline:0.1.0_c2b090043'

db/migrate/20250401141733_set_raw_location_on_ann_data_files.rb

@@ -0,0 +1,28 @@
+class SetRawLocationOnAnnDataFiles < Mongoid::Migration
+  def self.up
+    files = StudyFile.where(
+      file_type: 'AnnData', 'ann_data_file_info.reference_file' => false, parse_status: 'parsed'
+    ).select(&:is_raw_counts_file?)
+    files.each do |file|
+      adata = file.ann_data_file_info
+      exp_fragment = file.ann_data_file_info.find_fragment(data_type: :expression)
+      next if adata.raw_location.present? || exp_fragment.blank?
+
+      adata.raw_location = '.raw'
+      exp_fragment[:raw_location] = '.raw'
+      idx = file.ann_data_file_info.fragment_index_of(exp_fragment)
+      file.ann_data_file_info.data_fragments[idx] = exp_fragment
+      file.save
+    end
+  end
+
+  def self.down
+    files = StudyFile.where(
+      file_type: 'AnnData', 'ann_data_file_info.reference_file' => false, parse_status: 'parsed'
+    ).select(&:is_raw_counts_file?)
+    files.each do |file|
+      file.ann_data_file_info.raw_location = nil
+      file.save
+    end
+  end
+end