Merge pull request #2374 from broadinstitute/development

Release 1.111.1

Author: eweitz

.github/actions/setup-gcloud/action.yml

@@ -14,13 +14,13 @@ runs:
       shell: bash
       run: sudo apt-get update && sudo apt-get -y install jq
     - name: Auth to GCP
-      uses: google-github-actions/auth@v1
+      uses: google-github-actions/auth@v3
       with:
         project_id: ${{ inputs.google_cloud_project }}
         workload_identity_provider: "projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider"
         service_account: ${{ inputs.service_account_email }}
     - name: Set up Cloud SDK
-      uses: google-github-actions/setup-gcloud@v2
+      uses: google-github-actions/setup-gcloud@v3
       with:
         project_id: ${{ inputs.google_cloud_project }}
     - name: Authenticate docker

.github/workflows/build-and-publish-docker-image.yml

@@ -18,7 +18,7 @@ jobs:
     runs-on: ubuntu-24.04
     steps:
       - name: Check out repository code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
       - name: Configure gcloud
         uses: ./.github/actions/setup-gcloud
         with:

.github/workflows/deploy-to-production.yml

@@ -29,7 +29,7 @@ jobs:
       group: production-deployment
     steps:
       - name: Check out repository code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
       - name: Configure gcloud
         uses: ./.github/actions/setup-gcloud
         with:

.github/workflows/deploy-to-staging.yml

@@ -25,7 +25,7 @@ jobs:
       group: staging-deployment
     steps:
       - name: Check out repository code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
       - name: Configure gcloud
         uses: ./.github/actions/setup-gcloud
         with:

.github/workflows/run-all-tests.yml

@@ -17,8 +17,12 @@ jobs:
     runs-on: self-hosted
 
     steps:
+      - name: Set up node
+        uses: actions/setup-node@v6
+        with:
+          node-version: '24'
       - name: Check out repository code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
       - name: Configure gcloud
         uses: ./.github/actions/setup-gcloud
         with:

.github/workflows/run-orch-smoketest.yml

@@ -14,8 +14,12 @@ jobs:
     runs-on: self-hosted
 
     steps:
+      - name: Set up node
+        uses: actions/setup-node@v6
+        with:
+          node-version: '24'
       - name: Check out repository code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
       - name: Configure gcloud
         uses: ./.github/actions/setup-gcloud
         with:

app/javascript/components/upload/upload-utils.js

@@ -238,7 +238,8 @@ function validateNameUniqueness(file, allFiles, validationMessages) {
   const allOtherNames = allOtherFiles.map(f => f.name)
   // require 'isDirty' so we only show the error on the file being updated
   if (file.name && allOtherNames.includes(file.name) && file.isDirty) {
-    validationMessages.fileName = `A file named ${file.name} already exists in your study`
+    validationMessages.fileName = `A file named ${file.name} already exists in your study.  If you recently deleted ` +
+    `this file and are reusing it, please refresh the page to clear it from the cache.`
   }
 }
 

app/javascript/lib/dot-plot-precompute-patch.js

@@ -76,8 +76,17 @@
         // Series 1: percent expressing (for size) - will be scaled to 0-100
         // Data format: values[0] = mean_expression, values[1] = percent_expressing
         geneNames.forEach((gene, i) => {
-          const geneData = data.genes[gene]
-
+          // gotcha for pathway diagrams and precomputed dot plots
+          // this is because the pathway diagram may contain genes that were not included in the dot plot data
+          // (e.g. because they were not expressed)
+          let geneData = data.genes[gene]
+          if (!geneData) {
+            geneData = []
+            // If gene is missing from data, fill with NaN and 0 cells expressing
+            for (let j = 0; j < nCols; j++) {
+              geneData.push([NaN, 0])
+            }
+          }
           // Map data from original order to sorted order
           geneData.forEach((values, originalIndex) => {
             const sortedIndex = originalToSortedIndex[originalIndex]

app/javascript/lib/pathway-expression.js

@@ -21,6 +21,7 @@ import { renderDotPlot } from '~/components/visualization/DotPlot'
 import { getEligibleLabels } from '~/lib/cluster-utils'
 import { fetchMorpheusJson } from '~/lib/scp-api'
 import { getFeatureFlagsWithDefaults } from '~/providers/UserProvider'
+import { shouldUsePreprocessedData } from '~/components/visualization/DotPlot'
 
 // Denoise DevTools console log by not showing error that lacks user impact
 window.onerror = function(error) {
@@ -83,7 +84,7 @@ window.SCP.renderBackgroundDotPlotRegister = {}
 export async function renderBackgroundDotPlot(
   studyAccession, genes=[], cluster, annotation={},
   subsample, annotationValues, drawCallback,
-  topContainerSelector
+  topContainerSelector, usePreprocessed
 ) {
   const graphId = 'background-dot-plot'
   document.querySelector(`#${graphId}`)?.remove()
@@ -122,7 +123,9 @@ export async function renderBackgroundDotPlot(
       annotation.name,
       annotation.type,
       annotation.scope,
-      subsample
+      subsample,
+      false, // mock flag
+      usePreprocessed
     )
     dataset = results[0]
 
@@ -317,6 +320,8 @@ function mergeDotPlotMetrics(newMetrics, oldMetrics) {
 
 /** Color pathway gene nodes by expression */
 export async function renderPathwayExpression(studyAccession, cluster, annotation, label, labels) {
+  const flags = getFeatureFlagsWithDefaults()
+  const usePreprocessed = shouldUsePreprocessedData(flags, window.SCP?.exploreInfo)
   let allDotPlotMetrics = {}
 
   const pathwayGenes = getPathwayGenes()
@@ -337,10 +342,9 @@ export async function renderPathwayExpression(studyAccession, cluster, annotatio
     // the second render is for collapsed label-x-gene metrics (dotplot)
 
     numDraws += 1
-    if (numDraws === 1) {return}
+    if (numDraws === 1 && !usePreprocessed) {return}
 
     const dotPlotMetrics = getDotPlotMetrics(dotPlot)
-
     if (!dotPlotMetrics) {
       // Occurs upon resizing window, artifact of internal Morpheus handling
       // of pre-dot-plot heatmap matrix.  No user-facing impact.
@@ -367,7 +371,7 @@ export async function renderPathwayExpression(studyAccession, cluster, annotatio
       renderBackgroundDotPlot(
         studyAccession, dotPlotGeneBatches[numRenders], cluster, annotation,
         'All', annotationLabels, backgroundDotPlotDrawCallback,
-        '#_ideogramPathwayContainer'
+        '#_ideogramPathwayContainer', usePreprocessed
       )
     }
   }
@@ -378,7 +382,7 @@ export async function renderPathwayExpression(studyAccession, cluster, annotatio
   renderBackgroundDotPlot(
     studyAccession, dotPlotGeneBatches[0], cluster, annotation,
     'All', annotationLabels, backgroundDotPlotDrawCallback,
-    '#_ideogramPathwayContainer'
+    '#_ideogramPathwayContainer', usePreprocessed
   )
 }
 

app/javascript/lib/validation/validate-file.js

@@ -30,7 +30,8 @@ function validateFileName(file, studyFile, allStudyFiles, allowedFileExts=['*'])
   if (otherNames.includes(file.name) ||
     otherUploadFileNames.includes(file.name) ||
     otherLocalFileNames.includes(file.name)) {
-    const msg = `A file named ${file.name} already exists in your study`
+    const msg = `A file named ${file.name} already exists in your study.  If you recently deleted ` +
+      `this file and are reusing it, please refresh the page to clear it from the cache.`
     issues.push(['error', 'filename:duplicate', msg])
   }