Merge branch 'master' into mlin-fix-Fetch_SRA_to_BAM

dpark01 · web-flow · commit 5a856b179e87 · 2020-04-04T22:08:18.000-04:00
diff --git a/.travis.yml b/.travis.yml
@@ -8,6 +8,7 @@ env:
   global:
   - CACHE_DIR="$HOME/misc_cache"
   - MINICONDA_DIR="$HOME/miniconda"
+  - CONDA_DEFAULT_ENV="$HOME/misc_cache/default_env"
   - PYTHONIOENCODING=UTF8
   - BOTO_CONFIG=/dev/null # bogus value to override config on travis
   - DOCKER_REGISTRY="quay.io"
@@ -38,7 +39,8 @@ jobs:
       script:
         - set -e
         - travis/version-wdl-runtimes.sh
-        - travis/validate-wdl.sh
+        - travis/relative-wdl-paths.sh
+        - travis/validate-wdl-womtool.sh
         - travis/build-dx.sh
         - travis/tests-dx.sh
 
@@ -52,9 +54,25 @@ jobs:
       script:
         - set -e
         - travis/version-wdl-runtimes.sh
-        - travis/validate-wdl.sh
+        - travis/relative-wdl-paths.sh
+        - travis/validate-wdl-womtool.sh
         - travis/tests-cromwell.sh
 
+    - language: python
+      stage: build
+      env:
+        - TRAVIS_JOB=deploy_dockstore
+      install:
+        - set -e
+        - source travis/install-conda.sh
+        - conda install -y miniwdl shellcheck
+      script:
+        - set -e
+        - travis/version-wdl-runtimes.sh
+        - travis/validate-wdl-miniwdl.sh
+        - travis/flatten-wdls.sh
+        - echo "actual dockstore deploy not implemented yet"
+
     - language: python
       stage:    build
       env:      TRAVIS_JOB=build_docs
diff --git a/pipes/WDL/tasks/tasks_assembly.wdl b/pipes/WDL/tasks/tasks_assembly.wdl
@@ -30,7 +30,7 @@ task assemble {
             ${trim_clip_db} \
             ${sample_name}.assembly1-${assembler}.fasta \
             ${'--n_reads=' + trinity_n_reads} \
-     	      ${true='--alwaysSucceed' false="" always_succeed} \
+            ${true='--alwaysSucceed' false="" always_succeed} \
             --JVMmemory "$mem_in_mb"m \
             --outReads=${sample_name}.subsamp.bam \
             --loglevel=DEBUG
@@ -41,7 +41,7 @@ task assemble {
             ${trim_clip_db} \
             ${sample_name}.assembly1-${assembler}.fasta \
             ${'--nReads=' + spades_n_reads} \
-	          ${true="--alwaysSucceed" false="" always_succeed} \
+            ${true="--alwaysSucceed" false="" always_succeed} \
             ${'--minContigLen=' + spades_min_contig_len} \
             --memLimitGb $mem_in_gb \
             --outReads=${sample_name}.subsamp.bam \
@@ -55,15 +55,15 @@ task assemble {
             ${'--n_reads=' + trinity_n_reads} \
             --JVMmemory "$mem_in_mb"m \
             --outReads=${sample_name}.subsamp.bam \
-     	      ${true='--always_succeed' false='' always_succeed} \
+            ${true='--always_succeed' false='' always_succeed} \
             --loglevel=DEBUG
           assembly.py assemble_spades \
             ${reads_unmapped_bam} \
             ${trim_clip_db} \
             ${sample_name}.assembly1-${assembler}.fasta \
             --contigsUntrusted=${sample_name}.assembly1-trinity.fasta \
             ${'--nReads=' + spades_n_reads} \
-     	      ${true='--alwaysSucceed' false='' always_succeed} \
+            ${true='--alwaysSucceed' false='' always_succeed} \
             ${'--minContigLen=' + spades_min_contig_len} \
             --memLimitGb $mem_in_gb \
             --loglevel=DEBUG
diff --git a/pipes/WDL/workflows/demux_metag.wdl b/pipes/WDL/workflows/demux_metag.wdl
@@ -23,11 +23,15 @@ workflow demux_metag {
         assembler = "spades",
         reads_unmapped_bam = deplete.cleaned_bam
     }
+    call metagenomics.kaiju as kaiju {
+      input:
+        reads_unmapped_bam = raw_reads
+    }
   }
 
   call metagenomics.krakenuniq as kraken {
-    input:
-      reads_unmapped_bam = illumina_demux.raw_reads_unaligned_bams,
+      input:
+        reads_unmapped_bam = illumina_demux.raw_reads_unaligned_bams
   }
   call reports.aggregate_metagenomics_reports as metag_summary_report {
       input:
@@ -37,8 +41,4 @@ workflow demux_metag {
       input:
           spikein_count_txt = spikein.report
   }
-  call metagenomics.kaiju as kaiju {
-    input:
-      reads_unmapped_bam = illumina_demux.raw_reads_unaligned_bams,
-  }
 }
diff --git a/requirements-modules.txt b/requirements-modules.txt
@@ -1,4 +1,4 @@
-broadinstitute/viral-core=v2.0.14
-broadinstitute/viral-assemble=v2.0.14.0
-broadinstitute/viral-classify=v2.0.14.0
-broadinstitute/viral-phylo=v2.0.14.0
+broadinstitute/viral-core=v2.0.15
+broadinstitute/viral-assemble=v2.0.15.0
+broadinstitute/viral-classify=v2.0.15.0
+broadinstitute/viral-phylo=v2.0.15.0
diff --git a/test/input/WDL/test_inputs-assemble_denovo_with_deplete-local.json b/test/input/WDL/test_inputs-assemble_denovo_with_deplete-local.json
@@ -1,7 +1,7 @@
 {
   "assemble_denovo_with_deplete.deplete_taxa.raw_reads_unmapped_bam": "test/input/G5012.3.testreads.bam",
   "assemble_denovo_with_deplete.deplete_taxa.blastDbs": [
-	"test/input/5kb_human_from_chr6.fasta"
+    "test/input/5kb_human_from_chr6.fasta"
   ],
   "assemble_denovo_with_deplete.filter_to_taxon.lastal_db_fasta": "test/input/ebov-makona.fasta",
   "assemble_denovo_with_deplete.assemble.trim_clip_db": "test/input/clipDb.fasta",
diff --git a/travis/activate-conda.sh b/travis/activate-conda.sh
diff --git a/travis/flatten-wdls.sh b/travis/flatten-wdls.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+set -e -o pipefail
+
+FLAT_DIR=pipes/WDL/flattened
+mkdir -p $FLAT_DIR
+
+for workflow in pipes/WDL/workflows/*.wdl; do
+	wf_base=`basename $workflow`
+	out_fn="$FLAT_DIR/$wf_base"
+	echo "flattening $workflow to $out_fn"
+	travis/paste_wdl_imports.py -o $out_fn $workflow
+
+	### DEBUG
+	cat $out_fn
+done
diff --git a/travis/install-conda.sh b/travis/install-conda.sh
@@ -2,54 +2,34 @@
 set -e
 
 # the miniconda directory may exist if it has been restored from cache
-if [ -d "$MINICONDA_DIR" ] && [ -e "$MINICONDA_DIR/bin/conda" ]; then
+if [ -d "$MINICONDA_DIR" ] && [ -e "$MINICONDA_DIR/bin/conda" ] && [ -d "$CONDA_DEFAULT_ENV" ]; then
     echo "Miniconda install already present from cache: $MINICONDA_DIR"
-    if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
-        # on OSX we need to rely on the conda Python rather than the Travis-supplied system Python
-        # so conda has a higher precedence
-        export PATH="$MINICONDA_DIR/bin:$PATH"
-    else
-        export PATH="$MINICONDA_DIR/bin:$PATH"
-    fi
+    export PATH="$MINICONDA_DIR/bin:$PATH"
     hash -r
 else # if it does not exist, we need to install miniconda
-    rm -rf "$MINICONDA_DIR" # remove the directory in case we have an empty cached directory
-
-    if [[ "$TRAVIS_PYTHON_VERSION" == 2* ]]; then
-        if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
-            curl -S https://repo.continuum.io/miniconda/Miniconda2-latest-MacOSX-x86_64.sh > miniconda.sh;
-        else
-            curl -S https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh > miniconda.sh;
-        fi
-     else
-        if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
-            curl -S https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh > miniconda.sh;
-        else
-            curl -S https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh > miniconda.sh;
-        fi
-    fi
-
+    echo "Downloading conda to $MINICONDA_DIR"
+    rm -rf "$MINICONDA_DIR" "$CONDA_DEFAULT_ENV" # remove the directory in case we have an empty cached directory
+    curl -S https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh > miniconda.sh;
     bash miniconda.sh -b -p "$MINICONDA_DIR"
     chown -R "$USER" "$MINICONDA_DIR"
-    if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
-        # on OSX we need to rely on the conda Python rather than the Travis-supplied system Python
-        # so conda has a higher precedence
-        export PATH="$MINICONDA_DIR/bin:$PATH"
-    else
-        export PATH="$MINICONDA_DIR/bin:$PATH"
-    fi
+    export PATH="$MINICONDA_DIR/bin:$PATH"
     hash -r
-    conda install -y conda==4.6.14 # specify "conda update -c conda-canary conda" for pre-release conda
+    #echo "Installing conda"
+    #conda install -y conda==4.6.14 # specify "conda update -c conda-canary conda" for pre-release conda
     conda config --set always_yes yes --set changeps1 no --set remote_max_retries 6 #--set channel_priority strict
     conda config --add channels defaults
     conda config --add channels bioconda
     conda config --add channels conda-forge
-    conda config --add channels broad-viral
+    #conda config --add channels broad-viral
     conda config --show-sources # print channels
     # Use recommendations from https://github.com/bioconda/bioconda-recipes/issues/13774
     #conda update --quiet -y conda
     # conda config --set channel_priority strict
+    echo "Creating conda default env in $CONDA_DEFAULT_ENV"
+    conda create -y -m -p $CONDA_DEFAULT_ENV python="$TRAVIS_PYTHON_VERSION"
 fi
 
+echo "Activating conda environment: $CONDA_DEFAULT_ENV"
+source activate $CONDA_DEFAULT_ENV
 # update certs
 conda info -a # for debugging
diff --git a/travis/install-miniwdl.sh b/travis/install-miniwdl.sh
diff --git a/travis/install-pip.sh b/travis/install-pip.sh
diff --git a/travis/paste_wdl_imports.py b/travis/paste_wdl_imports.py
@@ -0,0 +1,125 @@
+#!/usr/bin/env python3
+
+"""
+Generate a standalone WDL document from a given workflow using imported tasks. Requires: miniwdl
+
+    python3 paste_wdl_imports.py [-o STANDALONE.wdl] WORKFLOW.wdl
+
+For each "call imported_namespace.task_name [as alias]" in the workflow, appends the task's source
+code with the task name changed to "imported_namespace__task_name", and rewrites the call to refer
+to this new name (keeping the original alias). Also blanks out the import statements.
+
+This is a quick-and-dirty hack with salient limitations:
+1. Can't handle imported structs
+2. Can't handle imported sub-workflows (tasks only)
+3. Each call, task, and import must begin on its own line of code with only whitespace preceding.
+
+Notes on these limitations:
+1. Structs would be tricky to get right, but probably doable. The problem is they can be imported
+   recursively (including diamond imports) with different names in every document.
+2. We can't do much about sub-workflows because a WDL document can only have one workflow.
+3. Stems from a crappy regex-based approach used because miniwdl WDL library lacks code generation
+"""
+
+import re
+import argparse
+import WDL
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-o",
+        metavar="STANDALONE.wdl",
+        help="output filename (default: standard output)",
+        default=None,
+    )
+    parser.add_argument(
+        "workflow_wdl", metavar="WORKFLOW.wdl", help="input workflow document with imports"
+    )
+    args = parser.parse_args()
+
+    # load workflow document
+    doc = WDL.load(args.workflow_wdl)
+    assert doc.workflow, "document has no workflow"
+
+    # run SetParents to facilitate getting from a called task to its containing document
+    WDL.Walker.SetParents()(doc)
+
+    # original document lines for surgery
+    doc_lines = doc.source_text.split("\n")
+
+    # for each call
+    tasks_processed = set()
+    for call in calls(doc.workflow):
+        if len(call.callee_id) > 1:  # imported
+            # create name for pasted task
+            new_task_name = "__".join(call.callee_id)
+            assert isinstance(call.callee, WDL.Task), "can't import sub-workflows"
+            # rewrite the call with the new task name
+            doc_lines[call.pos.line - 1] = rewrite_line(
+                doc_lines[call.pos.line - 1],
+                "call",
+                f"{new_task_name} as {call.name}",
+                old_name="[0-9A-Za-z_\\.]+(\\s+as\\s+[0-9A-Za-z_]+)?",
+            )
+            if new_task_name not in tasks_processed:
+                task_lines = task_source_lines(call.callee)
+                task_lines[0] = rewrite_line(task_lines[0], "task", new_task_name)
+                doc_lines += ["\n"] + task_lines + ["\n"]
+                tasks_processed.add(new_task_name)
+
+    # blank out the imports
+    for imp in doc.imports:
+        for ln in range(imp.pos.line - 1, imp.pos.end_line):
+            doc_lines[ln] = ""
+
+    # print output
+    if args.o:
+        with open(args.o, "w") as outfile:
+            for line in doc_lines:
+                print(line, file=outfile)
+    else:
+        for line in doc_lines:
+            print(line)
+
+
+def calls(element):
+    # Yield each Call in the workflow, including those nested withis scatter/conditional sections
+    for ch in element.children:
+        if isinstance(ch, WDL.Call):
+            yield ch
+        elif isinstance(ch, WDL.WorkflowSection):
+            yield from calls(ch)
+
+
+def rewrite_line(line, keyword, new_name, old_name="[0-9A-Za-z_\\.]+"):
+    # given a line like:
+    #   call OLD_NAME {
+    # replace OLD_NAME with new_name
+    parts = re.fullmatch(f"(?P<front>\\s*{keyword}\\s+){old_name}(?P<back>.*)", line)
+    assert parts, f"{keyword} should start on its own source line: {line}"
+    return parts.group("front") + new_name + parts.group("back")
+
+
+def task_source_lines(task):
+    # copy the source lines of the given task and change the task name
+    task_doc = task
+    while not isinstance(task_doc, WDL.Document):
+        task_doc = task_doc.parent
+    if not hasattr(task_doc, "source_lines"):
+        setattr(task_doc, "source_lines", task_doc.source_text.split("\n"))
+    first_line = task_doc.source_lines[task.pos.line - 1][task.pos.column - 1 :]
+    assert first_line.startswith("task")
+    assert task.pos.end_line > task.pos.line, "expected task to span multiple lines"
+    ans = (
+        [first_line]
+        + task_doc.source_lines[task.pos.line : (task.pos.end_line - 1)]
+        + [task_doc.source_lines[task.pos.end_line - 1][: task.pos.end_column]]
+    )
+    assert ans[-1][-1] == "}"
+    return ans
+
+
+if __name__ == "__main__":
+    main()
diff --git a/travis/relative-wdl-paths.sh b/travis/relative-wdl-paths.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+set -e -o pipefail
+
+echo Replacing relative paths to same directory
+sed -i -- 's|import \"../tasks/|import \"|g' pipes/WDL/workflows/*.wdl
diff --git a/travis/validate-wdl-miniwdl.sh b/travis/validate-wdl-miniwdl.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+set -e -o pipefail
+
+for workflow in pipes/WDL/workflows/*.wdl; do
+  echo "validating $workflow"
+  miniwdl check $workflow
+done
diff --git a/travis/validate-wdl-womtool.sh b/travis/validate-wdl-womtool.sh
diff --git a/travis/version-wdl-runtimes.sh b/travis/version-wdl-runtimes.sh
@@ -8,6 +8,4 @@ while IFS='=' read module version; do
 	NEW_TAG="$module:$STRIP_VERSION"
 	echo Replacing $OLD_TAG with $NEW_TAG in all task WDL files
 	sed -i -- "s|$OLD_TAG|$NEW_TAG|g" pipes/WDL/tasks/*.wdl
-	echo Replacing relative paths to same directory
-	sed -i -- 's|import \"../tasks/|import \"|g' pipes/WDL/workflows/*.wdl
 done < $MODULE_VERSIONS

Original file line number	Diff line number	Diff line change
`@@ -23,11 +23,15 @@ workflow demux_metag {`
`23`	`23`	`assembler = "spades",`
`24`	`24`	`reads_unmapped_bam = deplete.cleaned_bam`
`25`	`25`	`}`
	`26`	`+ call metagenomics.kaiju as kaiju {`
	`27`	`+ input:`
	`28`	`+ reads_unmapped_bam = raw_reads`
	`29`	`+ }`
`26`	`30`	`}`
`27`	`31`
`28`	`32`	`call metagenomics.krakenuniq as kraken {`
`29`		`- input:`
`30`		`- reads_unmapped_bam = illumina_demux.raw_reads_unaligned_bams,`
	`33`	`+ input:`
	`34`	`+ reads_unmapped_bam = illumina_demux.raw_reads_unaligned_bams`
`31`	`35`	`}`
`32`	`36`	`call reports.aggregate_metagenomics_reports as metag_summary_report {`
`33`	`37`	`input:`
`@@ -37,8 +41,4 @@ workflow demux_metag {`
`37`	`41`	`input:`
`38`	`42`	`spikein_count_txt = spikein.report`
`39`	`43`	`}`
`40`		`- call metagenomics.kaiju as kaiju {`
`41`		`- input:`
`42`		`- reads_unmapped_bam = illumina_demux.raw_reads_unaligned_bams,`
`43`		`- }`
`44`	`44`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,7 +1,7 @@`
`1`	`1`	`{`
`2`	`2`	`"assemble_denovo_with_deplete.deplete_taxa.raw_reads_unmapped_bam": "test/input/G5012.3.testreads.bam",`
`3`	`3`	`"assemble_denovo_with_deplete.deplete_taxa.blastDbs": [`
`4`		`- "test/input/5kb_human_from_chr6.fasta"`
	`4`	`+ "test/input/5kb_human_from_chr6.fasta"`
`5`	`5`	`],`
`6`	`6`	`"assemble_denovo_with_deplete.filter_to_taxon.lastal_db_fasta": "test/input/ebov-makona.fasta",`
`7`	`7`	`"assemble_denovo_with_deplete.assemble.trim_clip_db": "test/input/clipDb.fasta",`