Merge pull request #2235 from broadinstitute/development

Release 1.93.0

Author: bistline

Gemfile

@@ -88,6 +88,8 @@ gem 'ostruct'
 gem 'logger'
 gem 'benchmark'
 gem 'drb'
+gem 'reline'
+gem 'irb'
 
 group :development, :test do
   # Access an IRB console on exception pages or by using <%= console %> in views

Gemfile.lock

@@ -247,6 +247,10 @@ GEM
     image_processing (1.12.2)
       mini_magick (>= 4.9.5, < 5)
       ruby-vips (>= 2.0.17, < 3)
+    io-console (0.8.0)
+    irb (1.14.3)
+      rdoc (>= 4.0.0)
+      reline (>= 0.4.2)
     jbuilder (2.11.2)
       activesupport (>= 5.0.0)
     jquery-fileupload-rails (1.0.0)
@@ -424,6 +428,8 @@ GEM
     rdoc (6.6.3.1)
       psych (>= 4.0.0)
     regexp_parser (2.6.0)
+    reline (0.6.0)
+      io-console (~> 0.5)
     representable (3.2.0)
       declarative (< 0.1.0)
       trailblazer-option (>= 0.1.1, < 0.2.0)
@@ -580,6 +586,7 @@ DEPENDENCIES
   google-cloud-bigquery
   google-cloud-storage
   googleauth
+  irb
   jbuilder (~> 2.7)
   jquery-datatables-rails!
   jquery-fileupload-rails
@@ -610,6 +617,7 @@ DEPENDENCIES
   rack-brotli
   rack-mini-profiler
   rails (= 6.1.7.9)
+  reline
   rest-client
   rubocop
   rubocop-rails

app/controllers/api/v1/study_files_controller.rb

@@ -749,7 +749,7 @@ def study_file_params
           ],
           expression_file_info_attributes: [
             :_id, :_destroy, :library_preparation_protocol, :units, :biosample_input_type, :modality, :is_raw_counts,
-            raw_counts_associations: []
+            :raw_location, raw_counts_associations: []
           ],
           heatmap_file_info_attributes: [:id, :_destroy, :custom_scaling, :color_min, :color_max, :legend_label],
           cluster_form_info_attributes: [
@@ -758,8 +758,8 @@ def study_file_params
             :external_link_description, spatial_cluster_associations: []
           ],
           metadata_form_info_attributes: [:_id, :use_metadata_convention, :description],
-          extra_expression_form_info_attributes: [:_id, :taxon_id, :description, :y_axis_label],
-          ann_data_file_info_attributes: [:_id, :reference_file, :data_fragments],
+          extra_expression_form_info_attributes: [:_id, :taxon_id, :description, :y_axis_label, :raw_location],
+          ann_data_file_info_attributes: [:_id, :reference_file, :data_fragments, :raw_location],
           differential_expression_file_info_attributes: [
             :_id, :clustering_association, :annotation_name, :annotation_scope, :computational_method,
             :gene_header, :group_header, :comparison_group_header,

app/javascript/components/explore/DifferentialExpressionPanel.jsx

@@ -762,6 +762,7 @@ export default function DifferentialExpressionPanel({
           countsByLabelForDe={countsByLabelForDe}
           deObjects={deObjects}
           setDeFilePath={setDeFilePath}
+          isAuthorDe={isAuthorDe}
           deGroupB={deGroupB}
           setDeGroupB={setDeGroupB}
           hasOneVsRestDe={hasOneVsRestDe}
@@ -851,7 +852,7 @@ export default function DifferentialExpressionPanel({
           bucketId={bucketId}
           deFilePath={deFilePath}
           handleClear={handleClear}
-          isAuthorDe={hasPairwiseDe}
+          isAuthorDe={isAuthorDe}
           sizeMetric={sizeMetric}
           significanceMetric={significanceMetric}
           deFacets={deFacets}

app/javascript/components/upload/AnnDataExpressionStep.jsx

@@ -10,6 +10,7 @@ const DEFAULT_NEW_PROCESSED_FILE = {
     is_raw_counts: false,
     biosample_input_type: 'Whole cell',
     modality: 'Transcriptomic: unbiased',
+    raw_location: '',
     raw_counts_associations: []
   },
   file_type: 'Expression Matrix'

app/javascript/components/upload/ExpressionFileForm.jsx

@@ -8,6 +8,9 @@ import ExpandableFileForm from './ExpandableFileForm'
 
 import { TextFormField } from './form-components'
 import { findBundleChildren, validateFile } from './upload-utils'
+import { faQuestionCircle } from '@fortawesome/free-solid-svg-icons'
+import { OverlayTrigger, Popover } from 'react-bootstrap'
+import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'
 
 const REQUIRED_FIELDS = [{ label: 'species', propertyName: 'taxon_id' },
   { label: 'Biosample input type', propertyName: 'expression_file_info.biosample_input_type' },
@@ -19,6 +22,9 @@ const RAW_COUNTS_REQUIRED_FIELDS = REQUIRED_FIELDS.concat([{
 const PROCESSED_ASSOCIATION_FIELD = [
   { label: 'Associated raw counts files', propertyName: 'expression_file_info.raw_counts_associations' }
 ]
+const RAW_LOCATION_FIELD = [
+  { label: 'Raw count data location', propertyName: 'raw_location' },
+]
 
 /** renders a form for editing/uploading an expression file (raw or processed) and any bundle children */
 export default function ExpressionFileForm({
@@ -49,6 +55,10 @@ export default function ExpressionFileForm({
   if (rawCountsRequired && !isRawCountsFile ) {
     requiredFields = requiredFields.concat(PROCESSED_ASSOCIATION_FIELD)
   }
+  const requireLocation = (rawCountsRequired || isRawCountsFile) && isAnnDataExperience
+  if (requireLocation) {
+    requiredFields = requiredFields.concat(RAW_LOCATION_FIELD)
+  }
   const validationMessages = validateFile({ file, allFiles, allowedFileExts, requiredFields, isAnnDataExperience })
 
   const associatedRawCounts = !isAnnDataExperience && file.expression_file_info.raw_counts_associations.map(id => ({
@@ -61,9 +71,45 @@ export default function ExpressionFileForm({
     setShowRawCountsUnits(rawCountsVal)
   }
 
+  /** create the tooltip and message for the .obsm key name section */
+  function rawSlotMessage() {
+    const rawSlotToolTip = <span>
+      <OverlayTrigger
+        trigger={['hover', 'focus']}
+        rootClose placement="top"
+        delayHide={1500}
+        overlay={rawSlotHelpContent()}>
+        <span> Raw count data location * <FontAwesomeIcon icon={faQuestionCircle}/></span>
+      </OverlayTrigger>
+    </span>
+
+    return <span >
+      {rawSlotToolTip}
+    </span>
+  }
+
+  /** gets the popup message to describe .obsm keys */
+  function rawSlotHelpContent() {
+    const layersLink = <a href="https://anndata.readthedocs.io/en/latest/generated/anndata.AnnData.layers.html"
+                          target="_blank">layers</a>
+    const rawLink = <a href="https://anndata.readthedocs.io/en/latest/generated/anndata.AnnData.raw.html"
+                                target="_blank">.raw</a>
+    return <Popover id="cluster-obsm-key-name-popover" className="tooltip-wide">
+      <div>
+        Location of raw count data in your AnnData file. This can be the raw slot ({rawLink}) or the name of a layer in
+        the {layersLink} section.
+      </div>
+    </Popover>
+  }
+
   return <ExpandableFileForm {...{
-    file, allFiles, updateFile, saveFile,
-    allowedFileExts, deleteFile, validationMessages, bucketName, isInitiallyExpanded, isAnnDataExperience
+    file,
+    allFiles,
+    updateFile,
+    saveFile,
+    allowedFileExts,
+    deleteFile,
+    validationMessages, bucketName, isInitiallyExpanded, isAnnDataExperience
   }}>
     {!isAnnDataExperience &&
     <div className="form-group">
@@ -123,25 +169,32 @@ export default function ExpressionFileForm({
     { isAnnDataExperience &&
       <div className="row">
         <div className="form-radio col-sm-4">
-          <label className="labeled-select">I have raw count data in the <strong>adata.raw</strong> slot</label>
+          <label className="labeled-select">I have raw count data</label>
           <label className="sublabel">
             <input type="radio"
                    name={`anndata-raw-counts-${file._id}`}
                    value="true"
                    checked={isRawCountsFile}
-                   onChange={e => toggleIsRawCounts(true) } />
+                   onChange={e => toggleIsRawCounts(true)}/>
             &nbsp;Yes
           </label>
           <label className="sublabel">
             <input type="radio"
                    name={`anndata-raw-counts-${file._id}`}
                    value="false"
                    checked={!isRawCountsFile}
-                   onChange={e => toggleIsRawCounts(false) }/>
+                   onChange={e => toggleIsRawCounts(false)}/>
             &nbsp;No
           </label>
         </div>
-        {showRawCountsUnits && <div className="col-sm-8">
+        {requireLocation && <div className="col-sm-4">
+          <TextFormField label={rawSlotMessage()}
+                         fieldName="raw_location"
+                         file={file}
+                         updateFile={updateFile}
+                         placeholderText='Specify .raw or name of layer'/></div>
+        }
+        { showRawCountsUnits && <div className="col-sm-4">
           <ExpressionFileInfoSelect label="Units *"
                                     propertyName="units"
                                     rawOptions={fileMenuOptions.units}

app/javascript/components/visualization/controls/DifferentialExpressionGroupPicker.jsx

@@ -320,7 +320,7 @@ export function PairwiseDifferentialExpressionGroupLists({
 export function PairwiseDifferentialExpressionGroupPicker({
   bucketId, clusterName, annotation, deGenes, deGroup, setDeGroup,
   setDeGenes, countsByLabelForDe, deObjects, setDeFilePath,
-  deGroupB, setDeGroupB, hasOneVsRestDe, significanceMetric
+  deGroupB, setDeGroupB, hasOneVsRestDe, significanceMetric, isAuthorDe
 }) {
   const groups = getLegendSortedLabels(countsByLabelForDe)
 
@@ -343,7 +343,6 @@ export function PairwiseDifferentialExpressionGroupPicker({
 
     setDeFilePath(deFilePath)
 
-    const isAuthorDe = true // SCP doesn't currently automatically compute pairwise DE
     const deGenes = await fetchDeGenes(bucketId, deFilePath, isAuthorDe)
     setDeGenes(deGenes)
   }
@@ -441,7 +440,6 @@ export function OneVsRestDifferentialExpressionGroupPicker({
     const deFilePath = basePath + deFileName
 
     setDeFilePath(deFilePath)
-
     const deGenes = await fetchDeGenes(bucketId, deFilePath, isAuthorDe)
 
     setDeGroup(newGroup)

app/javascript/lib/validation/validate-anndata.js

@@ -137,10 +137,16 @@ async function checkOntologyLabelsAndIds(key, ontologies, groups) {
   const rawUniques = Array.from(labelIdPairs)
 
   rawUniques.map(r => {
-    const [id, label] = r.split(' || ')
+    let [id, label] = r.split(' || ')
     const ontologyShortNameLc = id.split(/[_:]/)[0].toLowerCase()
     const ontology = ontologies[ontologyShortNameLc]
 
+    if (id.includes(':')) {
+      // Convert colon to underscore for ontology lookup
+      const idParts = id.split(':')
+      id = `${idParts[0]}_${idParts[1]}`
+    }
+
     if (!(id in ontology)) {
       // Register invalid ontology ID
       const msg = `Invalid ontology ID: ${id}`

app/lib/differential_expression_service.rb

@@ -134,6 +134,7 @@ def self.run_differential_expression_job(cluster_group, study, user, annotation_
     metadata_url = study_file.is_viz_anndata? ?
                      RequestUtils.data_fragment_url(study_file, 'metadata') :
                      study.metadata_file.gs_url
+    cluster_group_id = cluster_group.id
     # begin assembling parameters
     de_params = {
       annotation_name:,
@@ -143,10 +144,12 @@ def self.run_differential_expression_job(cluster_group, study, user, annotation_
       group2:,
       annotation_file: annotation_scope == 'cluster' ? cluster_url : metadata_url,
       cluster_file: cluster_url,
-      cluster_name: cluster_group.name
+      cluster_name: set_cluster_name(study, cluster_group, annotation_name, annotation_scope),
+      cluster_group_id:
     }
     raw_matrix = ClusterVizService.raw_matrix_for_cluster_cells(study, cluster_group)
     de_params[:matrix_file_path] = raw_matrix.gs_url
+    de_params[:matrix_file_id] = raw_matrix.id
     if raw_matrix.file_type == 'MM Coordinate Matrix'
       de_params[:matrix_file_type] = 'mtx'
       # we know bundle exists and is completed as :raw_matrix_for_cluster_cells will throw an exception if it isn't
@@ -158,6 +161,7 @@ def self.run_differential_expression_job(cluster_group, study, user, annotation_
     elsif raw_matrix.file_type == 'AnnData'
       de_params[:matrix_file_type] = 'h5ad'
       de_params[:file_size] = raw_matrix.upload_file_size
+      de_params[:raw_location] = raw_matrix.ann_data_file_info.raw_location
     else
       de_params[:matrix_file_type] = 'dense'
     end
@@ -287,6 +291,21 @@ def self.annotation_eligible?(name)
     ALLOWED_ANNOTS =~ name && EXCLUDED_ANNOTS !~ name
   end
 
+  # determine if the requested cluster/annotation has an existing result object
+  # used for setting cluster name or in pairwise DE when merging in new results
+  #
+  # * *params*
+  #   - +study+            (Study) => Study in which results exist
+  #   - +cluster_group+    (ClusterGroup) => Clustering object to source name/file from
+  #   - +annotation_name+  (String) => Name of requested annotation
+  #   - +annotation_scope+ (String) => Scope of requested annotation ('study' or 'cluster')
+  #
+  # * *returns*
+  #   - (DifferentialExpressionResult, nil)
+  def self.find_existing_result(study, cluster_group, annotation_name, annotation_scope)
+    DifferentialExpressionResult.find_by(study:, cluster_group:, annotation_name:, annotation_scope:)
+  end
+
   # determine if a study already has DE results for an annotation, taking scope into account
   # cluster-based annotations must match to the specified cluster in the annotation object
   # for study-wide annotations, return true if any results exist, regardless of cluster as this indicates that DE
@@ -309,6 +328,21 @@ def self.results_exist?(study, annotation)
     ).exists?
   end
 
+  # helper to set cluster_name when existing results are present
+  # this prevents result file URLs breaking because the cluster has been renamed at some point
+  #
+  # * *params*
+  #   - +study+            (Study) => Study in which results exist
+  #   - +cluster_group+    (ClusterGroup) => Clustering object to source name/file from
+  #   - +annotation_name+  (String) => Name of requested annotation
+  #   - +annotation_scope+ (String) => Scope of requested annotation ('study' or 'cluster')
+  #
+  # * *returns*
+  #   - (String)
+  def self.set_cluster_name(study, cluster_group, annotation_name, annotation_scope)
+    find_existing_result(study, cluster_group, annotation_name, annotation_scope)&.cluster_name || cluster_group.name
+  end
+
   # determine if a study meets the requirements for differential expression:
   # 1. public
   # 2. has clustering/metadata
@@ -383,12 +417,15 @@ def self.validate_annotation(cluster_group, study, annotation_name, annotation_s
     elsif pairwise
       missing = [group1, group2] - annotation[:values]
       raise ArgumentError, "#{annotation_name} does not contain '#{missing.join(', ')}'" if missing.any?
+
       cell_count = {
-        "#{group1}" => cells_by_label[group1].count,
-        "#{group2}" => cells_by_label[group2].count
+        group1.to_s => cells_by_label[group1]&.count.to_i,
+        group2.to_s => cells_by_label[group2]&.count.to_i
       }.keep_if { |_, c| c < 2 }
-      raise ArgumentError,
-            "#{cell_count.keys.join(', ')} does not have enough cells represented in #{identifier}" if cell_count.any?
+      if cell_count.any?
+        raise ArgumentError, "#{cell_count.keys.join(', ')} does not have enough cells represented in #{identifier} " \
+          "for #{cluster_group.name}"
+      end
     end
   end
 

app/lib/file_parse_service.rb

@@ -112,13 +112,15 @@ def self.run_parse_job(study_file, study, user, reparse: false, persist_on_fail:
             )
           elsif study_file.needs_raw_counts_extraction?
             params_object = AnnDataIngestParameters.new(
-              anndata_file: study_file.gs_url, extract: %w[raw_counts], obsm_keys: nil,
+              anndata_file: study_file.gs_url, extract: %w[raw_counts],
+              raw_location: study_file.ann_data_file_info.raw_location, obsm_keys: nil,
               file_size: study_file.upload_file_size
             )
           else
             params_object = AnnDataIngestParameters.new(
               anndata_file: study_file.gs_url, obsm_keys: study_file.ann_data_file_info.obsm_key_names,
-              file_size: study_file.upload_file_size, extract_raw_counts: study_file.is_raw_counts_file?
+              file_size: study_file.upload_file_size, extract_raw_counts: study_file.is_raw_counts_file?,
+              raw_location: study_file.ann_data_file_info.raw_location
             )
           end
           # TODO extract and parse Raw Exp Data (SCP-4710)