RELEASE: staging -> master

.dockstore.yml

@@ -47,6 +47,10 @@ workflows:
     subclass: WDL
     primaryDescriptorPath: /all_of_us/admixture/convert_vcf_to_plink_bed.wdl
 
+  - name: CreateVcfIndex
+    subclass: WDL
+    primaryDescriptorPath: /all_of_us/phasing/CreateVcfIndex.wdl
+
   - name: CramToUnmappedBams
     subclass: WDL
     primaryDescriptorPath: /pipelines/wdl/reprocessing/cram_to_unmapped_bams/CramToUnmappedBams.wdl
@@ -76,6 +80,10 @@ workflows:
     subclass: WDL
     primaryDescriptorPath: /all_of_us/phasing/filter_and_qc_variants.wdl
 
+  - name: RemovePhasedSamples
+    subclass: WDL
+    primaryDescriptorPath: /all_of_us/phasing/remove_phased_samples.wdl
+
   - name: FixItFelixAndVariantCall
     subclass: WDL
     primaryDescriptorPath: /all_of_us/cmrg/FixItFelixAndVariantCall.wdl
@@ -92,6 +100,10 @@ workflows:
     subclass: WDL
     primaryDescriptorPath: /pipelines/wdl/glimpse/low_pass_imputation/Glimpse2LowPassImputationBatch.wdl
 
+  - name: Glimpse2LowPassImputationQuotaConsumed
+    subclass: WDL
+    primaryDescriptorPath: /pipelines/wdl/glimpse/low_pass_imputation/Glimpse2LowPassImputationQuotaConsumed.wdl
+
   - name: Glimpse2SVImputation
     subclass: WDL
     primaryDescriptorPath: /pipelines/wdl/glimpse/sv_imputation/Glimpse2SVImputation.wdl
@@ -100,9 +112,13 @@ workflows:
     subclass: WDL
     primaryDescriptorPath: /pipelines/wdl/glimpse/sv_imputation/Glimpse2SVImputationBatch.wdl
 
-  - name: Glimpse2LowPassImputationQuotaConsumed
+  - name: Glimpse2SVImputationQC
     subclass: WDL
-    primaryDescriptorPath: /pipelines/wdl/glimpse/low_pass_imputation/Glimpse2LowPassImputationQuotaConsumed.wdl
+    primaryDescriptorPath: /pipelines/wdl/glimpse/sv_imputation/input_qc/Glimpse2SVImputationQC.wdl
+
+  - name: Glimpse2SVImputationQuotaConsumed
+    subclass: WDL
+    primaryDescriptorPath: /pipelines/wdl/glimpse/sv_imputation/Glimpse2SVImputationQuotaConsumed.wdl
 
   - name: GetWgsMedianCoverage
     subclass: WDL
@@ -320,6 +336,14 @@ workflows:
     subclass: WDL
     primaryDescriptorPath: /verification/test-wdls/TestGlimpse2LowPassImputationQC.wdl
 
+  - name: TestGlimpse2SVImputation
+    subclass: WDL
+    primaryDescriptorPath: /verification/test-wdls/TestGlimpse2SVImputation.wdl
+
+  - name: TestGlimpse2SVImputationQC
+    subclass: WDL
+    primaryDescriptorPath: /verification/test-wdls/TestGlimpse2SVImputationQC.wdl
+
   - name: TestIlluminaGenotypingArray
     subclass: WDL
     primaryDescriptorPath: /verification/test-wdls/TestIlluminaGenotypingArray.wdl

.github/workflows/test_atac.yml

@@ -26,6 +26,8 @@ on:
       - '.github/workflows/test_atac.yml'
       - '.github/workflows/warp_test_workflow.yml'
       - 'scripts/firecloud_api/firecloud_api.py'
+      # dependencies imported by the test WDL but previously unwatched
+      - 'pipelines/wdl/peak_calling/PeakCalling.wdl'
 
 
   # Allows you to run this workflow manually from the Actions tab

.github/workflows/test_exome_germline_single_sample.yml

@@ -35,6 +35,10 @@ on:
       - '.github/workflows/test_exome_germline_single_sample.yml'
       - '.github/workflows/warp_test_workflow.yml'
       - 'scripts/firecloud_api/firecloud_api.py'
+      # dependencies imported by the test WDL but previously unwatched
+      - 'tasks/wdl/Alignment.wdl'
+      - 'tasks/wdl/DragmapAlignment.wdl'
+      - 'tasks/wdl/SplitLargeReadGroup.wdl'
 
 
   # Allows you to run this workflow manually from the Actions tab

.github/workflows/test_exome_reprocessing.yml

@@ -30,14 +30,18 @@ on:
      - 'verification/VerifyExomeReprocessing.wdl'
      - 'verification/VerifyGermlineSingleSample.wdl'
       # test WDL and its dependencies
-     - 'verification/test-wdls/TestCramToUnmappedBams.wdl'
+     - 'verification/test-wdls/TestExomeReprocessing.wdl'
      - 'verification/VerifyMetrics.wdl'
      - 'verification/VerifyTasks.wdl'
      - 'tasks/wdl/TerraCopyFilesFromCloudToCloud.wdl'
       # this file, the subworkflow file, and the firecloud_api script
      - '.github/workflows/test_exome_reprocessing.yml'
      - '.github/workflows/warp_test_workflow.yml'
      - 'scripts/firecloud_api/firecloud_api.py'
+     # dependencies imported by the test WDL but previously unwatched
+     - 'tasks/wdl/Alignment.wdl'
+     - 'tasks/wdl/DragmapAlignment.wdl'
+     - 'tasks/wdl/SplitLargeReadGroup.wdl'
 
 
   # Allows you to run this workflow manually from the Actions tab

.github/workflows/test_illumina_genotyping_array.yml

@@ -24,6 +24,9 @@ on:
       - '.github/workflows/test_illumina_genotyping_array.yml'
       - '.github/workflows/warp_test_workflow.yml'
       - 'scripts/firecloud_api/firecloud_api.py'
+      # dependencies imported by the test WDL but previously unwatched
+      - 'verification/VerifyMetrics.wdl'
+      - 'verification/VerifyTasks.wdl'
 
 
   # Allows you to run this workflow manually from the Actions tab

.github/workflows/test_imputation.yml

@@ -24,6 +24,8 @@ on:
       - '.github/workflows/test_imputation.yml'
       - '.github/workflows/warp_test_workflow.yml'
       - 'scripts/firecloud_api/firecloud_api.py'
+      # dependencies imported by the test WDL but previously unwatched
+      - 'verification/VerifyTasks.wdl'
 
 
   # Allows you to run this workflow manually from the Actions tab

.github/workflows/test_imputation_beagle.yml

@@ -20,6 +20,8 @@ on:
       - 'tasks/wdl/TerraCopyFilesFromCloudToCloud.wdl'
       - '.github/workflows/test_imputation_beagle.yml'
       - '.github/workflows/warp_test_workflow.yml'
+      # dependencies imported by the test WDL but previously unwatched
+      - 'verification/VerifyTasks.wdl'
 
 
   # Allows you to run this workflow manually from the Actions tab

.github/workflows/test_multiome.yml

@@ -34,6 +34,9 @@ on:
       - '.github/workflows/test_multiome.yml'
       - '.github/workflows/warp_test_workflow.yml'
       - 'scripts/firecloud_api/firecloud_api.py'
+      # dependencies imported by the test WDL but previously unwatched
+      - 'pipelines/wdl/atac/atac.wdl'
+      - 'pipelines/wdl/peak_calling/PeakCalling.wdl'
 
 
   # Allows you to run this workflow manually from the Actions tab

.github/workflows/test_pairedtag.yml

@@ -33,6 +33,9 @@ on:
       - '.github/workflows/test_pairedtag.yml'
       - '.github/workflows/warp_test_workflow.yml'
       - 'scripts/firecloud_api/firecloud_api.py'
+      # dependencies imported by the test WDL but previously unwatched
+      - 'pipelines/wdl/atac/atac.wdl'
+      - 'pipelines/wdl/peak_calling/PeakCalling.wdl'
 
 
   # Allows you to run this workflow manually from the Actions tab

.github/workflows/test_slidetags.yml

@@ -29,6 +29,10 @@ on:
       - '.github/workflows/test_slidetags.yml'
       - '.github/workflows/warp_test_workflow.yml'
       - 'scripts/firecloud_api/firecloud_api.py'
+      # dependencies imported by the test WDL but previously unwatched
+      - 'pipelines/wdl/optimus/Optimus.wdl'
+      - 'tasks/wdl/FastqProcessing.wdl'
+      - 'tasks/wdl/RunEmptyDrops.wdl'
 
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:

.github/workflows/test_snm3c.yml

@@ -22,6 +22,8 @@ on:
       - '.github/workflows/test_snm3c.yml'
       - '.github/workflows/warp_test_workflow.yml'
       - 'scripts/firecloud_api/firecloud_api.py'
+      # dependencies imported by the test WDL but previously unwatched
+      - 'verification/VerifyTasks.wdl'
 
 
   # Allows you to run this workflow manually from the Actions tab

.github/workflows/test_sv_imputation.yml

@@ -0,0 +1,79 @@
+name: Test Glimpse2SVImputation
+
+# Controls when the workflow will run
+on:
+  pull_request:
+    branches: [ "develop", "staging", "master" ]
+    # Only run if files in these paths changed:
+    ####################################
+    # SET PIPELINE SPECIFIC PATHS HERE #
+    ####################################
+    paths:
+      - 'pipelines/wdl/glimpse/sv_imputation/**'
+      - '!pipelines/wdl/glimpse/sv_imputation/input_qc/**'
+      - 'tasks/wdl/Glimpse2SVImputationTasks.wdl'
+      - 'verification/VerifyGlimpse2SVImputation.wdl'
+      - 'verification/test-wdls/TestGlimpse2SVImputation.wdl'
+      - 'verification/VerifyTasks.wdl'
+      - 'tasks/wdl/Utilities.wdl'
+      - 'tasks/wdl/TerraCopyFilesFromCloudToCloud.wdl'
+      - '.github/workflows/test_sv_imputation.yml'
+      - '.github/workflows/warp_test_workflow.yml'
+
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+    inputs:
+      useCallCache:
+        description: 'Use call cache (default: true)'
+        required: false
+        default: "true"
+      updateTruth:
+        description: 'Update truth files (default: false)'
+        required: false
+        default: "false"
+      testType:
+        description: 'Specify the type of test (Plumbing or Scientific)'
+        required: false
+        type: choice
+        options:
+          - Plumbing
+          - Scientific
+      truthBranch:
+        description: 'Specify the branch for truth files (default: master)'
+        required: false
+        default: "master"
+
+env:
+  # pipeline configuration
+  PIPELINE_NAME: TestGlimpse2SVImputation
+  DOCKSTORE_PIPELINE_NAME: Glimpse2SVImputation
+  PIPELINE_DIR: "pipelines/wdl/glimpse/sv_imputation/"
+
+  # workspace configuration
+  TESTING_WORKSPACE: WARP Tests
+  WORKSPACE_NAMESPACE: warp-pipelines
+
+  # service account configuration
+  SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+  USER: pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com
+
+
+jobs:
+  TestGlimpse2SVImputation:
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+      actions: write
+    uses: ./.github/workflows/warp_test_workflow.yml
+    with:
+      pipeline_name: TestGlimpse2SVImputation
+      dockstore_pipeline_name: Glimpse2SVImputation
+      pipeline_dir: pipelines/wdl/glimpse/sv_imputation
+      use_call_cache: ${{ github.event.inputs.useCallCache || 'true' }}
+      update_truth: ${{ github.event.inputs.updateTruth || 'false' }}
+      test_type: ${{ github.event.inputs.testType }}
+      truth_branch: ${{ github.event.inputs.truthBranch || 'master' }}
+    secrets:
+      PDT_TESTER_SA_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+      DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}

.github/workflows/test_sv_imputation_qc.yml

@@ -0,0 +1,76 @@
+name: Test Glimpse2SVImputationQC
+
+# Controls when the workflow will run
+on:
+  pull_request:
+    branches: [ "develop", "staging", "master" ]
+    # Only run if files in these paths changed:
+    ####################################
+    # SET PIPELINE SPECIFIC PATHS HERE #
+    ####################################
+    paths:
+      - 'pipelines/wdl/glimpse/sv_imputation/input_qc/**'
+      - 'verification/VerifyGlimpse2SVImputationQC.wdl'
+      - 'verification/test-wdls/TestGlimpse2SVImputationQC.wdl'
+      - 'tasks/wdl/Utilities.wdl'
+      - 'tasks/wdl/TerraCopyFilesFromCloudToCloud.wdl'
+      - '.github/workflows/test_sv_imputation_qc.yml'
+      - '.github/workflows/warp_test_workflow.yml'
+
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+    inputs:
+      useCallCache:
+        description: 'Use call cache (default: true)'
+        required: false
+        default: "true"
+      updateTruth:
+        description: 'Update truth files (default: false)'
+        required: false
+        default: "false"
+      testType:
+        description: 'Specify the type of test (Plumbing or Scientific)'
+        required: false
+        type: choice
+        options:
+          - Plumbing
+          - Scientific
+      truthBranch:
+        description: 'Specify the branch for truth files (default: master)'
+        required: false
+        default: "master"
+
+env:
+  # pipeline configuration
+  PIPELINE_NAME: TestGlimpse2SVImputationQC
+  DOCKSTORE_PIPELINE_NAME: Glimpse2SVImputationQC
+  PIPELINE_DIR: "pipelines/wdl/glimpse/sv_imputation/input_qc"
+
+  # workspace configuration
+  TESTING_WORKSPACE: WARP Tests
+  WORKSPACE_NAMESPACE: warp-pipelines
+
+  # service account configuration
+  SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+  USER: pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com
+
+
+jobs:
+  TestGlimpse2SVImputationQC:
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+      actions: write
+    uses: ./.github/workflows/warp_test_workflow.yml
+    with:
+      pipeline_name: TestGlimpse2SVImputationQC
+      dockstore_pipeline_name: Glimpse2SVImputationQC
+      pipeline_dir: pipelines/wdl/glimpse/sv_imputation/input_qc
+      use_call_cache: ${{ github.event.inputs.useCallCache || 'true' }}
+      update_truth: ${{ github.event.inputs.updateTruth || 'false' }}
+      test_type: ${{ github.event.inputs.testType }}
+      truth_branch: ${{ github.event.inputs.truthBranch || 'master' }}
+    secrets:
+      PDT_TESTER_SA_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+      DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}

.github/workflows/test_ultima_genomics_whole_genome_germline.yml

@@ -35,6 +35,8 @@ on:
       - '.github/workflows/test_ultima_genomics_whole_genome_germline.yml'
       - '.github/workflows/warp_test_workflow.yml'
       - 'scripts/firecloud_api/firecloud_api.py'
+      # dependencies imported by the test WDL but previously unwatched
+      - 'pipelines/wdl/dna_seq/somatic/single_sample/ugwgs/UltimaGenomicsWholeGenomeCramOnly.wdl'
 
 
   # Allows you to run this workflow manually from the Actions tab

.github/workflows/test_whole_genome_germline_single_sample.yml

@@ -35,6 +35,8 @@ on:
       - '.github/workflows/test_whole_genome_germline_single_sample.yml'
       - '.github/workflows/warp_test_workflow.yml'
       - 'scripts/firecloud_api/firecloud_api.py'
+      # dependencies imported by the test WDL but previously unwatched
+      - 'tasks/wdl/UnmappedBamToAlignedBam.wdl'
 
 
   # Allows you to run this workflow manually from the Actions tab

.github/workflows/test_whole_genome_reprocessing.yml

@@ -38,6 +38,8 @@ on:
       - '.github/workflows/test_whole_genome_reprocessing.yml'
       - '.github/workflows/warp_test_workflow.yml'
       - 'scripts/firecloud_api/firecloud_api.py'
+      # dependencies imported by the test WDL but previously unwatched
+      - 'tasks/wdl/UnmappedBamToAlignedBam.wdl'
 
 
   # Allows you to run this workflow manually from the Actions tab

.github/workflows/update_versions_file.yml

@@ -34,14 +34,13 @@ jobs:
           echo "Current directory: "
           pwd
           ls -lht
-          echo -e "Pipeline Name\tVersion\tDate of Last Commit" > unsorted_pipeline_versions.txt
-          find pipelines -name "*.changelog.md" -type f | while read -r file; do
+          echo -e "Pipeline Name\tVersion\tDate of Last Commit" > pipeline_versions.txt
+          find pipelines -name "*.changelog.md" -type f -not -path "*/deprecated/*" | while read -r file; do
             pipeline_name=$(basename "$file" .changelog.md)
             version=$(head -n 1 "$file" | sed 's/#//')
             last_commit_date=$(sed -n '2p' "$file" | sed 's/([^)]*)//g')
-            echo -e "$pipeline_name\t$version\t$last_commit_date" >> unsorted_pipeline_versions.txt
-          done
-          sort -k1,1 unsorted_pipeline_versions.txt > pipeline_versions.txt
+            echo -e "$pipeline_name\t$version\t$last_commit_date"
+          done | sort -k1,1 >> pipeline_versions.txt
 
       # This step commits and pushes the changes to the repository
       - name: Commit and push changes