Merge pull request #56 from nextstrain/fix-species-ingest

Add species to ingest/nextstrain-automation

Author: joverlee521

.github/workflows/ingest-to-phylogenetic.yaml

@@ -41,22 +41,22 @@ jobs:
     permissions:
       id-token: write
     uses: nextstrain/.github/.github/workflows/pathogen-repo-build.yaml@master
-    secrets: inherit  
+    secrets: inherit
     with:
       runtime: docker
       env: |
         NEXTSTRAIN_DOCKER_IMAGE: ${{ inputs.image }}
       run: |
         nextstrain build ingest \
           upload_all \
-          --configfile build-configs/nextstrain-automation/config.yaml 
+          --configfile build-configs/nextstrain-automation/config.yaml
       # Specifying artifact name to differentiate ingest build outputs from the phylogenetic build outputs
       artifact-name: ingest-output
       artifact-paths: |
         ingest/benchmarks/
         ingest/logs/
         ingest/.snakemake/log/
-  
+
 
   # Check if ingest results include new data by checking for the cache
   # of the file with the results' Metadata.sh256sum (which should have been added within upload-to-s3)
@@ -74,7 +74,7 @@ jobs:
           AWS_DEFAULT_REGION: ${{ vars.AWS_DEFAULT_REGION }}
         run: |
           s3_urls=(
-            "s3://nextstrain-data/files/workflows/ebola/metadata.tsv.zst"
+            "s3://nextstrain-data/files/workflows/ebola/ebov/metadata.tsv.zst"
           )
 
           # Code below is modified from ingest/upload-to-s3
@@ -102,7 +102,7 @@ jobs:
 
   phylogenetic:
     needs: [check-new-data]
-    if: ${{ needs.check-new-data.outputs.cache-hit != 'true' }}
+    if: ${{ github.event_name == 'workflow_dispatch' || needs.check-new-data.outputs.cache-hit != 'true' }}
     permissions:
       id-token: write
     uses: nextstrain/.github/.github/workflows/pathogen-repo-build.yaml@master
@@ -120,4 +120,4 @@ jobs:
       artifact-paths: |
         phylogenetic/benchmarks/
         phylogenetic/logs/
-        phylogenetic/.snakemake/log/
+        phylogenetic/.snakemake/log/

ingest/Snakefile

@@ -8,7 +8,7 @@ workdir: workflow.current_basedir
 # Use default configuration values. Override with Snakemake's --configfile/--config options.
 configfile: "defaults/config.yaml"
 
-SPECIES = ['ebov', 'bdbv', 'sudv']
+SPECIES = config["species"]
 
 # This is the default rule that Snakemake will run when there are no specified targets.
 # The default output of the ingest workflow is usually the curated metadata and sequences.

ingest/build-configs/nextstrain-automation/config.yaml

@@ -1,6 +1,10 @@
 # This configuration file should contain all required configuration parameters
 # for the ingest workflow to run with additional Nextstrain automation rules.
 
+# Only run ebov because the bdbv/sudv workflows are not set up to pull data from S3
+species:
+  - ebov
+
 # Custom rules to run as part of the Nextstrain automated workflow
 # The paths should be relative to the ingest directory.
 custom_rules:
@@ -15,10 +19,10 @@ s3_dst: "s3://nextstrain-data/files/workflows/ebola"
 
 # Mapping of files to upload
 files_to_upload:
-  metadata.tsv.zst: results/metadata.tsv
-  metadata_open.tsv.zst: results/metadata_open.tsv
-  sequences.fasta.zst: results/sequences.fasta
-  sequences_open.fasta.zst: results/sequences_open.fasta
-  alignment.fasta.zst: results/alignment.fasta
-  alignment_open.fasta.zst: results/alignment_open.fasta
-  translations.zip: results/translations.zip # note - no 'open' translations yet
+  ebov/metadata.tsv.zst: results/ebov/metadata.tsv
+  ebov/metadata_open.tsv.zst: results/ebov/metadata_open.tsv
+  ebov/sequences.fasta.zst: results/ebov/sequences.fasta
+  ebov/sequences_open.fasta.zst: results/ebov/sequences_open.fasta
+  ebov/alignment.fasta.zst: results/ebov/alignment.fasta
+  ebov/alignment_open.fasta.zst: results/ebov/alignment_open.fasta
+  ebov/translations.zip: results/ebov/translations.zip # note - no 'open' translations yet

ingest/defaults/config.yaml

@@ -3,6 +3,10 @@
 #
 # Define optional config parameters with their default values here so that users
 # do not have to dig through the workflows to figure out the default values
+species:
+  - ebov
+  - bdbv
+  - sudv
 
 ppx_fetch:
   ebov: # zaire

phylogenetic/all-outbreaks/Snakefile

@@ -5,13 +5,13 @@ Note: This does not conform to our modern config-based workflow.
 """
 
 inputs = {
-    "metadata": "s3://nextstrain-data/files/workflows/ebola/metadata.tsv.zst",
-    "alignment": "s3://nextstrain-data/files/workflows/ebola/alignment.fasta.zst",
-    "translations": "s3://nextstrain-data/files/workflows/ebola/translations.zip",
+    "metadata": "s3://nextstrain-data/files/workflows/ebola/ebov/metadata.tsv.zst",
+    "alignment": "s3://nextstrain-data/files/workflows/ebola/ebov/alignment.fasta.zst",
+    "translations": "s3://nextstrain-data/files/workflows/ebola/ebov/translations.zip",
 } if config.get('s3', False) is True else {
-    "metadata": "../ingest/results/metadata.tsv",
-    "alignment": "../ingest/results/alignment.fasta",
-    "translations": "../ingest/results/translations.zip",
+    "metadata": "../ingest/results/ebov/metadata.tsv",
+    "alignment": "../ingest/results/ebov/alignment.fasta",
+    "translations": "../ingest/results/ebov/translations.zip",
 }
 
 include: "../../shared/vendored/snakemake/remote_files.smk"
@@ -100,7 +100,7 @@ rule refine:
     shell:
         r"""
         exec &> >(tee {log:q})
-        
+
         augur refine \
             --tree {input.tree:q} \
             --keep-root \
@@ -119,7 +119,7 @@ rule unzip_translations:
         exec &> >(tee {log:q})
 
         mkdir -p results/translations
-        
+
         unzip -o {input[0]} -d results/translations
         """