Merge branch 'develop' into lk_aou_add_rna_documentation

Author: ekiernan

.dockstore.yml

@@ -4,11 +4,11 @@ workflows:
   - name: aggregate_rsem_results
     subclass: WDL
     primaryDescriptorPath: /all_of_us/rna_seq/aggregate_rsem_results.wdl
-  
+ 
   - name: AggregateSusieWorkflow
     subclass: WDL
     primaryDescriptorPath: /all_of_us/rna_seq/AggregateSusieWorkflow.wdl
-  
+ 
   - name: ArrayImputationQuotaConsumed
     subclass: WDL
     primaryDescriptorPath: /pipelines/wdl/arrays/imputation_beagle/ArrayImputationQuotaConsumed.wdl
@@ -85,19 +85,19 @@ workflows:
   - name: leafcutter_bam_to_junc
     subclass: WDL
     primaryDescriptorPath: /all_of_us/rna_seq/leafcutter_bam_to_junc.wdl
-  
+
   - name: leafcutter_cluster
     subclass: WDL
     primaryDescriptorPath: /all_of_us/rna_seq/leafcutter_cluster.wdl
-    
+
   - name: LocalizeAndConcatStargazer
     subclass: WDL
     primaryDescriptorPath: /all_of_us/pgx/LocalizeAndConcatStargazer.wdl
-  
+
   - name: MakeTable
     subclass: WDL
     primaryDescriptorPath: /all_of_us/hla/MakeTable.wdl
-    
+
   - name: mitochondria_pipeline
     subclass: WDL
     primaryDescriptorPath: /all_of_us/mitochondria/mitochondria_pipeline.wdl
@@ -139,23 +139,23 @@ workflows:
   - name: RNAWithUMIsPipeline
     subclass: WDL
     primaryDescriptorPath: /pipelines/wdl/rna_seq/RNAWithUMIsPipeline.wdl
-  
+
   - name: run_preprocess_admixture_est_rye
     subclass: WDL
     primaryDescriptorPath: /all_of_us/admixture/run_preprocess_admixture_est_rye.wdl
-  
+
   - name: run_admixture_est_rye
     subclass: WDL
     primaryDescriptorPath: /all_of_us/admixture/run_admixture_est_rye.wdl
-  
+
   - name: run_ancestry
     subclass: WDL
     primaryDescriptorPath: /all_of_us/ancestry/run_ancestry.wdl
 
   - name: run_relatedness
     subclass: WDL
     primaryDescriptorPath: /all_of_us/ancestry/run_relatedness.wdl
-  
+
   - name: run_sample_outlier_qc
     subclass: WDL
     primaryDescriptorPath: /all_of_us/ancestry/run_sample_outlier_qc.wdl
@@ -189,11 +189,11 @@ workflows:
 
     primaryDescriptorPath: /beta-pipelines/broad/Utilities/SplitMultisampleVCF/SplitMultisampleVCF.wdl
     readMePath: /beta-pipelines/broad/Utilities/SplitMultisampleVCF/README.md
-  
+
   - name: StargazerFromJointVCF
     subclass: WDL
     primaryDescriptorPath: /all_of_us/pgx/StargazerFromJointVCF.wdl
-  
+
 
   - name: SusieR_Workflow
     subclass: WDL

.github/workflows/warp_test_workflow.yml

@@ -211,6 +211,7 @@ jobs:
           done
 
           echo "Error: The Dockstore Commit Hash does not match the GitHub Commit Hash after 15 minutes of retries!"
+          
           exit 1
         env:
           GITHUB_COMMIT_HASH: ${{ env.GITHUB_COMMIT_HASH }}
@@ -251,19 +252,18 @@ jobs:
           TEST_TYPE="${{ env.testType }}"
           INPUTS_DIR="${{ inputs.pipeline_dir }}/test_inputs/$TEST_TYPE"
           echo "Running tests with test type: $TEST_TYPE"
-
-          TRUTH_PATH="gs://broad-gotc-test-storage/${{ inputs.dockstore_pipeline_name }}/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
-          echo "Truth path: $TRUTH_PATH"
           
           RESULTS_PATH="gs://pd-test-results/${{ inputs.dockstore_pipeline_name }}/results/$CURRENT_TIME"
 
+
           # Submit all jobs first and store their submission IDs
           for input_file in "$INPUTS_DIR"/*.json; do
-            test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
+            test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py \
                 --results_path "$RESULTS_PATH" \
                 --inputs_json "$input_file" \
                 --update_truth "$UPDATE_TRUTH_BOOL" \
-                --branch_name "$BRANCH_NAME" )
+                --branch_name "$TRUTH_BRANCH" \
+                --dockstore_pipeline_name "${{ inputs.dockstore_pipeline_name }}" )
             echo "Uploading the test input file: $test_input_file"
 
             # Create the submission_data.json file for this input_file

deprecated/pipelines/internal_rna_seq/test_inputs/Plumbing/SM-HFCB1_100ng_.65X_A3.hg19.fastqs.plumbing.json

@@ -4,8 +4,8 @@
 
   "BroadInternalRNAWithUMIs.reference_build": "hg19",
 
-  "BroadInternalRNAWithUMIs.r1_fastq": "gs://broad-gotc-test-storage/rna_seq/rna_with_umis/plumbing/inputs/SM-HFCB1_100ng_.65X_A3_read1_plumbing.fastq",
-  "BroadInternalRNAWithUMIs.r2_fastq": "gs://broad-gotc-test-storage/rna_seq/rna_with_umis/plumbing/inputs/SM-HFCB1_100ng_.65X_A3_read2_plumbing.fastq",
+  "BroadInternalRNAWithUMIs.r1_fastq": "gs://pd-test-storage-public/RNAWithUMIsPipeline/input/plumbing/SM-HFCB1_100ng_.65X_A3_read1_plumbing.fastq",
+  "BroadInternalRNAWithUMIs.r2_fastq": "gs://pd-test-storage-public/RNAWithUMIsPipeline/input/plumbing/SM-HFCB1_100ng_.65X_A3_read2_plumbing.fastq",
   "BroadInternalRNAWithUMIs.read1Structure": "3M2S146T",
   "BroadInternalRNAWithUMIs.read2Structure": "3M2S146T",
   "BroadInternalRNAWithUMIs.library_name": "SM-HFCB1_100ng_.65X_A3",

deprecated/pipelines/internal_rna_seq/test_inputs/Plumbing/SM-HFCB1_100ng_.65X_A3.hg38.fastqs.plumbing.json

@@ -4,8 +4,8 @@
 
   "BroadInternalRNAWithUMIs.reference_build": "hg38",
 
-  "BroadInternalRNAWithUMIs.r1_fastq": "gs://broad-gotc-test-storage/rna_seq/rna_with_umis/plumbing/inputs/SM-HFCB1_100ng_.65X_A3_read1_plumbing.fastq",
-  "BroadInternalRNAWithUMIs.r2_fastq": "gs://broad-gotc-test-storage/rna_seq/rna_with_umis/plumbing/inputs/SM-HFCB1_100ng_.65X_A3_read2_plumbing.fastq",
+  "BroadInternalRNAWithUMIs.r1_fastq": "gs://pd-test-storage-public/RNAWithUMIsPipeline/input/plumbing/SM-HFCB1_100ng_.65X_A3_read1_plumbing.fastq",
+  "BroadInternalRNAWithUMIs.r2_fastq": "gs://pd-test-storage-public/RNAWithUMIsPipeline/input/plumbing/SM-HFCB1_100ng_.65X_A3_read2_plumbing.fastq",
   "BroadInternalRNAWithUMIs.read1Structure": "3M2S146T",
   "BroadInternalRNAWithUMIs.read2Structure": "3M2S146T",
   "BroadInternalRNAWithUMIs.library_name": "SM-HFCB1_100ng_.65X_A3",

deprecated/pipelines/internal_rna_seq/test_inputs/Plumbing/SM-K4Y2X_350ng_.65X_D3.hg19.fastqs.plumbing.json

@@ -4,8 +4,8 @@
 
   "BroadInternalRNAWithUMIs.reference_build": "hg19",
 
-  "BroadInternalRNAWithUMIs.r1_fastq": "gs://broad-gotc-test-storage/rna_seq/rna_with_umis/plumbing/inputs/SM-K4Y2X_350ng_.65X_D3_read1_plumbing.fastq",
-  "BroadInternalRNAWithUMIs.r2_fastq": "gs://broad-gotc-test-storage/rna_seq/rna_with_umis/plumbing/inputs/SM-K4Y2X_350ng_.65X_D3_read2_plumbing.fastq",
+  "BroadInternalRNAWithUMIs.r1_fastq": "gs://pd-test-storage-private/RNAWithUMIsPipeline/input/plumbing/SM-K4Y2X_350ng_.65X_D3_read1_plumbing.fastq",
+  "BroadInternalRNAWithUMIs.r2_fastq": "gs://pd-test-storage-private/RNAWithUMIsPipeline/input/plumbing/SM-K4Y2X_350ng_.65X_D3_read2_plumbing.fastq",
   "BroadInternalRNAWithUMIs.read1Structure": "3M2S146T",
   "BroadInternalRNAWithUMIs.read2Structure": "3M2S146T",
   "BroadInternalRNAWithUMIs.library_name": "SM-K4Y2X_350ng_.65X_D3",

deprecated/pipelines/internal_rna_seq/test_inputs/Plumbing/SM-K4Y2X_350ng_.65X_D3.hg38.fastqs.plumbing.json

@@ -4,8 +4,8 @@
 
   "BroadInternalRNAWithUMIs.reference_build": "hg38",
 
-  "BroadInternalRNAWithUMIs.r1_fastq": "gs://broad-gotc-test-storage/rna_seq/rna_with_umis/plumbing/inputs/SM-K4Y2X_350ng_.65X_D3_read1_plumbing.fastq",
-  "BroadInternalRNAWithUMIs.r2_fastq": "gs://broad-gotc-test-storage/rna_seq/rna_with_umis/plumbing/inputs/SM-K4Y2X_350ng_.65X_D3_read2_plumbing.fastq",
+  "BroadInternalRNAWithUMIs.r1_fastq": "gs://pd-test-storage-private/RNAWithUMIsPipeline/input/plumbing/SM-K4Y2X_350ng_.65X_D3_read1_plumbing.fastq",
+  "BroadInternalRNAWithUMIs.r2_fastq": "gs://pd-test-storage-private/RNAWithUMIsPipeline/input/plumbing/SM-K4Y2X_350ng_.65X_D3_read2_plumbing.fastq",
   "BroadInternalRNAWithUMIs.read1Structure": "3M2S146T",
   "BroadInternalRNAWithUMIs.read2Structure": "3M2S146T",
   "BroadInternalRNAWithUMIs.library_name": "SM-K4Y2X_350ng_.65X_D3",

deprecated/pipelines/internal_rna_seq/test_inputs/Scientific/SM-HFCB1_100ng_.65X_A3.hg19.fastqs.json

@@ -4,8 +4,8 @@
 
   "BroadInternalRNAWithUMIs.reference_build": "hg19",
 
-  "BroadInternalRNAWithUMIs.r1_fastq": "gs://broad-gotc-test-storage/rna_seq/rna_with_umis/scientific/inputs/SM-HFCB1_100ng_.65X_A3_read1.fastq",
-  "BroadInternalRNAWithUMIs.r2_fastq": "gs://broad-gotc-test-storage/rna_seq/rna_with_umis/scientific/inputs/SM-HFCB1_100ng_.65X_A3_read2.fastq",
+  "BroadInternalRNAWithUMIs.r1_fastq": "gs://pd-test-storage-public/RNAWithUMIsPipeline/input/scientific/SM-HFCB1_100ng_.65X_A3_read1.fastq",
+  "BroadInternalRNAWithUMIs.r2_fastq": "gs://pd-test-storage-public/RNAWithUMIsPipeline/input/scientific/SM-HFCB1_100ng_.65X_A3_read2.fastq",
   "BroadInternalRNAWithUMIs.read1Structure": "3M2S146T",
   "BroadInternalRNAWithUMIs.read2Structure": "3M2S146T",
   "BroadInternalRNAWithUMIs.library_name": "SM-HFCB1_100ng_.65X_A3",

deprecated/pipelines/internal_rna_seq/test_inputs/Scientific/SM-HFCB1_100ng_.65X_A3.hg38.fastqs.json

@@ -4,8 +4,8 @@
 
   "BroadInternalRNAWithUMIs.reference_build": "hg38",
 
-  "BroadInternalRNAWithUMIs.r1_fastq": "gs://broad-gotc-test-storage/rna_seq/rna_with_umis/scientific/inputs/SM-HFCB1_100ng_.65X_A3_read1.fastq",
-  "BroadInternalRNAWithUMIs.r2_fastq": "gs://broad-gotc-test-storage/rna_seq/rna_with_umis/scientific/inputs/SM-HFCB1_100ng_.65X_A3_read2.fastq",
+  "BroadInternalRNAWithUMIs.r1_fastq": "gs://pd-test-storage-public/RNAWithUMIsPipeline/input/scientific/SM-HFCB1_100ng_.65X_A3_read1.fastq",
+  "BroadInternalRNAWithUMIs.r2_fastq": "gs://pd-test-storage-public/RNAWithUMIsPipeline/input/scientific/SM-HFCB1_100ng_.65X_A3_read2.fastq",
   "BroadInternalRNAWithUMIs.read1Structure": "3M2S146T",
   "BroadInternalRNAWithUMIs.read2Structure": "3M2S146T",
   "BroadInternalRNAWithUMIs.library_name": "SM-HFCB1_100ng_.65X_A3",

deprecated/pipelines/internal_rna_seq/test_inputs/Scientific/SM-K4Y2X_350ng_.65X_D3.hg19.fastqs.json

@@ -4,8 +4,8 @@
 
   "BroadInternalRNAWithUMIs.reference_build": "hg38",
 
-  "BroadInternalRNAWithUMIs.r1_fastq": "gs://broad-gotc-test-storage/rna_seq/rna_with_umis/scientific/inputs/SM-K4Y2X_350ng_.65X_D3_read1.fastq",
-  "BroadInternalRNAWithUMIs.r2_fastq": "gs://broad-gotc-test-storage/rna_seq/rna_with_umis/scientific/inputs/SM-K4Y2X_350ng_.65X_D3_read2.fastq",
+  "BroadInternalRNAWithUMIs.r1_fastq": "gs://pd-test-storage-private/RNAWithUMIsPipeline/input/scientific/SM-K4Y2X_350ng_.65X_D3_read1.fastq",
+  "BroadInternalRNAWithUMIs.r2_fastq": "gs://pd-test-storage-private/RNAWithUMIsPipeline/input/scientific/SM-K4Y2X_350ng_.65X_D3_read2.fastq",
   "BroadInternalRNAWithUMIs.read1Structure": "3M2S146T",
   "BroadInternalRNAWithUMIs.read2Structure": "3M2S146T",
   "BroadInternalRNAWithUMIs.library_name": "SM-K4Y2X_350ng_.65X_D3",

deprecated/pipelines/internal_rna_seq/test_inputs/Scientific/SM-K4Y2X_350ng_.65X_D3.hg38.fastqs.json

@@ -4,8 +4,8 @@
 
   "BroadInternalRNAWithUMIs.reference_build": "hg38",
 
-  "BroadInternalRNAWithUMIs.r1_fastq": "gs://broad-gotc-test-storage/rna_seq/rna_with_umis/scientific/inputs/SM-K4Y2X_350ng_.65X_D3_read1.fastq",
-  "BroadInternalRNAWithUMIs.r2_fastq": "gs://broad-gotc-test-storage/rna_seq/rna_with_umis/scientific/inputs/SM-K4Y2X_350ng_.65X_D3_read2.fastq",
+  "BroadInternalRNAWithUMIs.r1_fastq": "gs://pd-test-storage-private/RNAWithUMIsPipeline/input/scientific/SM-K4Y2X_350ng_.65X_D3_read1.fastq",
+  "BroadInternalRNAWithUMIs.r2_fastq": "gs://pd-test-storage-private/RNAWithUMIsPipeline/input/scientific/SM-K4Y2X_350ng_.65X_D3_read2.fastq",
   "BroadInternalRNAWithUMIs.read1Structure": "3M2S146T",
   "BroadInternalRNAWithUMIs.read2Structure": "3M2S146T",
   "BroadInternalRNAWithUMIs.library_name": "SM-K4Y2X_350ng_.65X_D3",