requester pay stuff

Author: SHuang-Broad

wdl/tasks/Utility/BAMutils.wdl

@@ -1041,7 +1041,6 @@ task SubsetBamToLocusLocal {
         interval_list_file:  "a Picard-style interval list file to subset reads with"
         interval_id:         "an ID string for representing the intervals in the interval list file"
         prefix: "prefix for output bam and bai file names"
-        bam: {localization_optional: true}
     }
 
     input {
@@ -1059,13 +1058,11 @@ task SubsetBamToLocusLocal {
 
     String subset_prefix = prefix + "." + interval_id
 
-    String local_bam = "/cromwell_root/~{basename(bam)}"
-
     command <<<
-        set -euxo pipefail
+    set -euxo pipefail
 
-        time gcloud storage cp ~{bam} ~{local_bam}
-        mv ~{bai} "~{local_bam}.bai" && touch "~{local_bam}.bai"
+        # guard against cases where the bai doesn't follow the naming convension of ".bam.bai"
+        if [[ -f "~{bam}.bai" ]]; then mv ~{bai} "~{bam}.bai"; fi
 
         # see man page for what '-M' means
         samtools view \
@@ -1074,7 +1071,7 @@ task SubsetBamToLocusLocal {
             -@ 1 \
             --write-index \
             -o "~{subset_prefix}.bam##idx##~{subset_prefix}.bam.bai" \
-            ~{local_bam} "~{local_bam}.bai" \
+            ~{bam} "~{bam}.bai" \
             ~{sep=" " intervals}
     >>>
 
@@ -1092,7 +1089,7 @@ task SubsetBamToLocusLocal {
         disk_gb:            disk_size,
         preemptible_tries:  0,
         max_retries:        0,
-        docker:             "us.gcr.io/broad-dsp-lrma/lr-gcloud-samtools:0.1.3"
+        docker:             "us.gcr.io/broad-dsp-lrma/lr-gcloud-samtools:0.1.23"
     }
     RuntimeAttr runtime_attr = select_first([runtime_attr_override, default_attr])
     runtime {

wdl/tasks/Utility/Utils.wdl

@@ -1526,42 +1526,60 @@ task SubsetBam {
         RuntimeAttr? runtime_attr_override
     }
 
+    String subset_prefix = prefix + "." + locus
+
     command <<<
+        # the way this works is the following:
+        # 0) relying on the re-auth.sh script to export the credentials
+        # 1) perform the remote sam-view subsetting in the background
+        # 2) listen to the PID of the background process, while re-auth every 1200 seconds
+        source /opt/re-auth.sh
         set -euxo pipefail
 
-        export GCS_OAUTH_TOKEN=$(gcloud auth application-default print-access-token)
-
         echo "false" > "samtools.failed.txt"
 
+        # see man page for what '-M' means
         samtools view \
             -bhX \
             -M \
             -@ 1 \
             --verbosity=8 \
             --write-index \
-            -o "~{prefix}.bam##idx##~{prefix}.bam.bai" \
+            -o "~{subset_prefix}.bam##idx##~{subset_prefix}.bam.bai" \
             ~{bam} ~{bai} \
-            ~{locus} \
-        || { echo "samtools seem to have failed"; echo "true" > "samtools.failed.txt"; exit 77; }
+            ~{locus} && exit 0 || { echo "samtools seem to have failed"; echo "true" > "samtools.failed.txt"; exit 77; } &
+        pid=$!
+
+        set +e
+        count=0
+        while true; do
+            sleep 1200 && date && source /opt/re-auth.sh
+            count=$(( count+1 ))
+            if [[ ${count} -gt 6 ]]; then echo "true" > "samtools.failed.txt" && exit 0; fi  # way too many attempts, get out
+            if ! pgrep -x -P $pid; then exit 0; fi
+        done
     >>>
 
     output {
-        File subset_bam = "~{prefix}.bam"
-        File subset_bai = "~{prefix}.bam.bai"
+        File subset_bam = "~{subset_prefix}.bam"
+        File subset_bai = "~{subset_prefix}.bam.bai"
         Boolean is_samtools_failed = read_boolean("samtools.failed.txt")
     }
 
     #########################
-    Int disk_size = if (0==disk_offset) then 4*ceil(size([bam, bai], "GB")) else disk_offset + ceil(size([bam, bai], "GB"))
+    # Int disk_size = if (0==disk_offset) then 4*ceil(size([bam, bai], "GB")) else disk_offset + ceil(size([bam, bai], "GB"))
+    Int min_disk = 10
+    Int proposal_disk = ceil(0.2 * size([bam, bai], "GiB"))
+    Int disk_size = if (proposal_disk<min_disk) then min_disk else proposal_disk # here we make one assumption that we aren't getting more than 10% of the whole genome
 
     RuntimeAttr default_attr = object {
-        cpu_cores:          1,
+        cpu_cores:          2,
         mem_gb:             10,
         disk_gb:            disk_size,
         boot_disk_gb:       10,
         preemptible_tries:  2,
         max_retries:        1,
-        docker:             "us.gcr.io/broad-dsp-lrma/lr-basic:0.1.1"
+        docker:             "us.gcr.io/broad-dsp-lrma/lr-gcloud-samtools:0.1.23"
     }
     RuntimeAttr runtime_attr = select_first([runtime_attr_override, default_attr])
     runtime {
@@ -1779,7 +1797,7 @@ task ResilientSubsetBam {
         boot_disk_gb:       10,
         preemptible_tries:  2,
         max_retries:        1,
-        docker:             "us.gcr.io/broad-dsp-lrma/lr-basic:0.1.1"
+        docker:             "us.gcr.io/broad-dsp-lrma/lr-gcloud-samtools:0.1.23"
     }
     RuntimeAttr runtime_attr = select_first([runtime_attr_override, default_attr])
     runtime {