Merge pull request #2292 from broadinstitute/development

Release 1.101.0

Author: bistline

Dockerfile

@@ -1,5 +1,5 @@
 # use SCP base Rails image, configure only project-specific items here
-FROM gcr.io/broad-singlecellportal-staging/rails-baseimage:3.0.0
+FROM gcr.io/broad-singlecellportal-staging/rails-baseimage:3.0.1
 
 # Set ruby version
 RUN bash -lc 'rvm --default use ruby-3.4.2'
@@ -8,8 +8,6 @@ RUN bash -lc 'rvm rvmrc warning ignore /home/app/webapp/Gemfile'
 # Set up project dir, install gems, set up script to migrate database and precompile static assets on run
 RUN mkdir /home/app/webapp
 RUN sudo chown app:app /home/app/webapp # fix permission issues in local development on MacOSX
-RUN gem update --system
-RUN gem install bundler
 COPY Gemfile /home/app/webapp/Gemfile
 COPY Gemfile.lock /home/app/webapp/Gemfile.lock
 WORKDIR /home/app/webapp

Gemfile.lock

@@ -331,11 +331,11 @@ GEM
       net-protocol
     netrc (0.11.0)
     nio4r (2.7.4)
-    nokogiri (1.18.8-arm64-darwin)
+    nokogiri (1.18.9-arm64-darwin)
       racc (~> 1.4)
-    nokogiri (1.18.8-x86_64-darwin)
+    nokogiri (1.18.9-x86_64-darwin)
       racc (~> 1.4)
-    nokogiri (1.18.8-x86_64-linux-gnu)
+    nokogiri (1.18.9-x86_64-linux-gnu)
       racc (~> 1.4)
     oauth2 (1.4.7)
       faraday (>= 0.8, < 2.0)
@@ -505,7 +505,7 @@ GEM
     systemu (2.6.5)
     test-unit (3.4.1)
       power_assert
-    thor (1.3.2)
+    thor (1.4.0)
     tilt (2.0.10)
     time_difference (0.5.0)
       activesupport
@@ -641,4 +641,4 @@ RUBY VERSION
    ruby 3.4.2p28
 
 BUNDLED WITH
-   2.6.2
+   2.7.1

app/controllers/api/v1/search_controller.rb

@@ -29,10 +29,10 @@ class SearchController < ApiBaseController
           parameter do
             key :name, :type
             key :in, :query
-            key :description, 'Type of query to perform (study- or cell-based)'
+            key :description, 'Type of query to perform (study- or gene-based)'
             key :required, true
             key :type, :string
-            key :enum, ['study', 'cell']
+            key :enum, %w[study gene]
           end
           parameter do
             key :name, :facets
@@ -94,6 +94,14 @@ class SearchController < ApiBaseController
             key :type, :string
             key :enum, [:recent, :popular]
           end
+          parameter do
+            key :name, :export
+            key :in, :query
+            key :description, 'Export results as a file'
+            key :required, false
+            key :type, :string
+            key :enum, %w[tsv]
+          end
           response 200 do
             key :description, 'Search parameters, Studies and StudyFiles'
             schema do
@@ -391,10 +399,17 @@ def index
         end
 
         @matching_accessions = @studies.map { |study| self.class.get_study_attribute(study, :accession) }
-
+        logger.info "studies_by_facet: #{@studies_by_facet}"
         logger.info "Final list of matching studies: #{@matching_accessions}"
         @results = @studies.paginate(page: params[:page], per_page: Study.per_page)
-        render json: search_results_obj, status: 200
+        if params[:export].present?
+          send_data results_text_export,
+                    filename: "scp_search_results_#{Time.now.strftime('%Y%m%d_%H%M%S')}.#{params[:export]}",
+                    status: :ok,
+                    x_sendfile: true
+        else
+          render json: search_results_obj, status: 200
+        end
       end
 
       swagger_path '/search/facets' do
@@ -657,7 +672,9 @@ def self.match_results_by_filter(search_result:, result_key:, facets:)
         else
           matching_facet[:filters].detect do |filter|
             filters = search_result[result_key].is_a?(Array) ? search_result[result_key] : [search_result[result_key]]
-            filters.include?(filter[:id]) || filters.include?(filter[:name])
+            filters.include?(filter[:id]) ||
+              filters.include?(filter[:name]) ||
+              filters.include?(filter[:id].gsub(/_/, ':')) # handle malformed IDs
           end
         end
       end

app/controllers/api/v1/study_search_results_objects.rb

@@ -8,6 +8,13 @@ module StudySearchResultsObjects
       # list of metadata names to include in cohort responses
       COHORT_METADATA = %w[disease__ontology_label organ__ontology_label species__ontology_label sex
                            library_preparation_protocol__ontology_label].freeze
+      # headers for TSV text export of search results
+      TEXT_HEADERS = [
+        'Study source', 'Accession', 'Name', 'Description', 'Public', 'Detached', 'Cell count', 'Gene count',
+        'Study URL', 'Disease', 'Organ', 'Species', 'Sex', 'Library preparation protocol', 'Facet matches',
+        'Term matches'
+      ].freeze
+      COMMA_SPACE = ', '.freeze
 
       def search_results_obj
         response_obj = {
@@ -112,6 +119,79 @@ def cohort_metadata(study)
         cohort_entries
       end
 
+      def results_text_export
+        lines = [TEXT_HEADERS.join("\t")]
+        @studies.each { |study| lines << study_text_export(study).join("\t") }
+        lines.join("\n")
+      end
+
+      def study_text_export(study)
+        if study.is_a?(Study)
+          metadata = cohort_metadata(study).with_indifferent_access
+          term_matches = study.search_weight(@term_list || [])
+          facet_data = @studies_by_facet&.[](study.accession) || {}
+          text_to_facet = @metadata_matches&.[](study.accession) || {}
+          facet_matches = Api::V1::StudySearchResultsObjects.merge_facet_matches(facet_data, text_to_facet)
+          inferred = @inferred_accessions&.include?(study.accession)
+          [
+            'SCP',
+            study.accession,
+            study.name,
+            study.description,
+            study.public,
+            study.detached,
+            study.cell_count,
+            study.gene_count,
+            result_url_for(study),
+            metadata[:disease].join(COMMA_SPACE),
+            metadata[:organ].join(COMMA_SPACE),
+            metadata[:species].join(COMMA_SPACE),
+            metadata[:sex].join(COMMA_SPACE),
+            metadata[:library_preparation_protocol].join(COMMA_SPACE),
+            Api::V1::StudySearchResultsObjects.facet_results_as_text(facet_matches),
+            inferred ? "inferred text match on #{@inferred_terms.join(COMMA_SPACE)}" : term_matches[:terms].keys.join(COMMA_SPACE)
+          ]
+        else
+          [
+            study[:hca_result] ? 'HCA' : 'TDR',
+            study[:accession],
+            study[:name],
+            study[:description].gsub(/\n/, ''),
+            true,
+            false,
+            0,
+            0,
+            result_url_for(study),
+            study.dig(:metadata, :disease).join(COMMA_SPACE),
+            study.dig(:metadata, :organ).join(COMMA_SPACE),
+            study.dig(:metadata, :species).join(COMMA_SPACE),
+            study.dig(:metadata, :sex).join(COMMA_SPACE),
+            study.dig(:metadata, :library_preparation_protocol).join(COMMA_SPACE),
+            Api::V1::StudySearchResultsObjects.facet_results_as_text(@studies_by_facet[study[:accession]]),
+            nil
+          ]
+        end
+      end
+
+      # returns a URL for the study, either SCP or HCA
+      def result_url_for(study)
+        if study.is_a?(Study)
+          view_study_url(accession: study.accession, study_name: study.url_safe_name)
+        else
+          "https://data.humancellatlas.org/explore/projects/#{study[:hca_project_id]}"
+        end
+      end
+
+      # flatten facet matches into a text string for export
+      def self.facet_results_as_text(facets)
+        entries = []
+        facets.delete(:facet_search_weight)
+        facets.each do |facet_name, filters|
+          entries << "#{facet_name}:#{filters.map { |f| f[:name] }.join('|')}"
+        end
+        entries.join(COMMA_SPACE)
+      end
+
       # merge in multiple facet match data objects into a single merged entity for a given study
       def self.merge_facet_matches(existing_data, new_data)
         study_data = existing_data || {}

app/controllers/site_controller.rb

@@ -695,7 +695,8 @@ def study_params
                                   :default_options => [:cluster, :annotation, :color_profile, :expression_label,
                                                        :deliver_emails, :cluster_point_size, :cluster_point_alpha,
                                                        :cluster_point_border, :precomputed_heatmap_label,
-                                                       :expression_sort, override_viz_limit_annotations: []],
+                                                       :expression_sort, override_viz_limit_annotations: [],
+                                                       cluster_order: [], spatial_order: []],
                                   study_shares_attributes: [:id, :_destroy, :email, :permission],
                                   study_detail_attributes: [:id, :full_description],
                                   reviewer_access_attributes: [:id, :expires_at],

app/controllers/studies_controller.rb

@@ -1039,7 +1039,8 @@ def default_options_params
     params.require(:study_default_options).permit(:cluster, :annotation, :color_profile, :expression_label,
                                                   :cluster_point_size, :cluster_point_alpha, :cluster_point_border,
                                                   :precomputed_heatmap_label, :expression_sort,
-                                                  override_viz_limit_annotations: [])
+                                                  override_viz_limit_annotations: [], cluster_order: [],
+                                                  spatial_order: [])
   end
 
   def set_file_types

app/javascript/components/search/results/ResultsExport.jsx

@@ -0,0 +1,31 @@
+import React, { useState } from 'react'
+import Button from 'react-bootstrap/lib/Button'
+import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'
+import { faFileExport } from '@fortawesome/free-solid-svg-icons'
+import { exportSearchResultsText } from '~/lib/scp-api'
+
+export default function ResultsExport({ studySearchState }) {
+  const hasResults = studySearchState?.results?.studies && studySearchState.results.studies.length > 0
+  const [exporting, setExporting] = useState(false)
+
+  /** export results to a file */
+  async function exportResults() {
+    setExporting(true)
+    await exportSearchResultsText(studySearchState.params).then(() => {
+      setExporting(false)
+    })
+  }
+
+  return (
+    <Button
+      onClick={async () => {await exportResults()}}
+      disabled={exporting || !hasResults}
+      data-testid="export-search-results-tsv"
+      data-analytics-name="export-search-results-tsv"
+      data-original-title="Export search results to TSV file"
+      data-toggle="tooltip"
+    >
+      <FontAwesomeIcon icon={faFileExport} /> {exporting ? 'Exporting...' : 'Export'}
+    </Button>
+  )
+}

app/javascript/components/search/results/ResultsPanel.jsx

@@ -51,7 +51,11 @@ const ResultsPanel = ({ studySearchState, studyComponent, noResultsDisplay, book
   } else if (results.studies && results.studies.length > 0) {
     panelContent = (
       <>
-        { <SearchQueryDisplay terms={results.termList} facets={results.facets} bookmarks={bookmarks}/> }
+        { <SearchQueryDisplay terms={results.termList}
+                              facets={results.facets}
+                              bookmarks={bookmarks}
+                              studySearchState={studySearchState}/> }
+        { }
         <StudyResults
           results={results}
           StudyComponent={studyComponent ? studyComponent : StudySearchResult}

app/javascript/components/search/results/SearchQueryDisplay.jsx

@@ -7,6 +7,7 @@ import _flatten from 'lodash/flatten'
 import { getDisplayNameForFacet } from '~/providers/SearchFacetProvider'
 import { SearchSelectionContext } from '~/providers/SearchSelectionProvider'
 import BookmarkManager from '~/components/bookmarks/BookmarkManager'
+import ResultsExport from '~/components/search/results/ResultsExport'
 
 /** joins texts by wrapping them in a span with itemClass className, and then
  * inserting spans with the joinText
@@ -88,7 +89,7 @@ export const ClearAllButton = () => {
 /** displays a summary of an executed search.
  * e.g. (Text contains (stomach)) AND (Metadata contains (organ: brain))
  */
-export default function SearchQueryDisplay({ terms, facets, bookmarks }) {
+export default function SearchQueryDisplay({ terms, facets, bookmarks, studySearchState }) {
   const hasFacets = facets && facets.length > 0
   const hasTerms = terms && terms.length > 0
   if (!hasFacets && !hasTerms) {
@@ -133,6 +134,7 @@ export default function SearchQueryDisplay({ terms, facets, bookmarks }) {
       <FontAwesomeIcon icon={faSearch}/>: <span className="query-text">
         {termsDisplay}{facetsDisplay}
       </span> <ClearAllButton/>
+      <ResultsExport studySearchState={studySearchState} />
       <BookmarkManager bookmarks={bookmarks} />
     </div>
   )

app/javascript/lib/scp-api.jsx

@@ -510,6 +510,34 @@ export async function downloadBucketFile(bucketId, filePath) {
   document.body.removeChild(element)
 }
 
+export async function exportSearchResultsText(searchParams, mock=false) {
+  searchParams.export = 'tsv'
+  const path = `/search?${buildSearchQueryString('study', searchParams)}`
+  const init = {
+    method: 'GET',
+    headers: {
+      Authorization: `Bearer ${getOAuthToken()}`
+    }
+  }
+  const [searchResults, perfTimes] = await scpApi(path, init, mock, false, false)
+  const fileName = searchResults.headers.get('Content-Disposition').split('"')[1]
+  const dataBlob = await searchResults.blob()
+
+  // Create an element with an anchor link and connect this to the blob
+  const element = document.createElement('a')
+  element.href = URL.createObjectURL(dataBlob)
+
+  // name the file and indicate it should download
+  element.download = fileName
+
+  // Simulate clicking the link resulting in downloading the file
+  document.body.appendChild(element)
+  element.click()
+
+  // Cleanup
+  document.body.removeChild(element)
+}
+
 /**
  * Returns initial content for the "Explore" tab in Study Overview
  *
@@ -898,7 +926,7 @@ export async function fetchSearch(type, searchParams, mock=false) {
 export function buildSearchQueryString(type, searchParams) {
   const facetsParam = buildFacetQueryString(searchParams.facets)
 
-  const params = ['page', 'order', 'terms', 'external', 'preset', 'genes', 'genePage']
+  const params = ['page', 'order', 'terms', 'external', 'export', 'preset', 'genes', 'genePage']
   let otherParamString = params.map(param => {
     return searchParams[param] ? `&${param}=${searchParams[param]}` : ''
   }).join('')