add apptainer workflow scripts (#43)

* add apptainer workflow scripts

* temporarily build container on test branch

* remove running workflow on test branch once container has been built

* add apptainer profile and bump version

* correct container reference

* add missing parentheses around profile

Author: taranewman

.github/scripts/build_container_images_wave.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+mkdir -p artifacts
+mkdir -p wave_images
+
+for env_yaml in environments/*.yml; do
+    image_name=$(head -n 1 $env_yaml | cut -d ' ' -f 2)
+    echo "building image ${image_name} from file ${env_yaml}..."
+    wave \
+	--conda-file ${env_yaml} \
+	--singularity \
+	--freeze \
+	--await \
+	--output json \
+	| python -m json.tool \
+	| tee wave_images/${image_name}.json
+    echo "done building image ${image_name}"
+    cp wave_images/${image_name}.json artifacts/
+done

.github/scripts/install_apptainer.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -eo pipefail
+wget https://raw.githubusercontent.com/apptainer/apptainer/main/tools/install-unprivileged.sh
+chmod +x install-unprivileged.sh
+mkdir -p /opt/apptainer
+./install-unprivileged.sh /opt/apptainer
+echo "/opt/apptainer/bin" >> $GITHUB_PATH

.github/scripts/install_wave-cli.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -eo pipefail
+wget https://github.com/seqeralabs/wave-cli/releases/download/v1.4.1/wave-1.4.1-linux-x86_64
+mv wave-1.4.1-linux-x86_64 wave
+chmod +x wave
+mkdir -p /opt/wave/bin
+mv wave /opt/wave/bin
+echo "/opt/wave/bin" >> $GITHUB_PATH

.github/scripts/push_container_images_wave.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python
+
+import argparse
+import glob
+import json
+import os
+import subprocess
+
+
+def pull_image(image_url, destination_image_file):
+    """
+    Pull the image file from external server
+    """
+    apptainer_pull_cmd = [
+        "apptainer",
+        "pull",
+        destination_image_file,
+        image_url,
+    ]
+    subprocess.run(apptainer_pull_cmd)
+
+
+def push_image(source_image_file, image_url):
+    """
+    Push apptainer image to destination image repository
+    """
+
+    apptainer_push_cmd = [
+        "apptainer",
+        "push",
+        source_image_file,
+        image_url,
+    ]
+    subprocess.run(apptainer_push_cmd)
+
+
+def main(args):
+    repo_owner = os.environ['GITHUB_REPOSITORY_OWNER'].lower()
+    
+    wave_jsons = glob.glob(os.path.join(args.wave_jsons_dir, "*.json"))
+    for wave_json in wave_jsons:
+        with open(wave_json, 'r') as f:
+            w = json.load(f)
+            pull_image_url = w['containerImage']
+            image_name_with_version = pull_image_url.split('/')[-1]
+            image_name, image_version = image_name_with_version.split(':')
+            pull_destination = os.path.join(args.images_dir, f"{image_name}--{image_version}.img")
+            pull_image(pull_image_url, pull_destination)
+
+            push_image_url = f"oras://ghcr.io/{repo_owner}/{image_name}:{image_version}"
+            push_image(pull_destination, push_image_url)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--wave-jsons-dir')
+    parser.add_argument('--images-dir')
+    args = parser.parse_args()
+    main(args)

.github/workflows/build_and_push_containers.yml

@@ -0,0 +1,37 @@
+name: Build and Push Container Images
+
+on:
+  push:
+    tags:
+      - v*
+  workflow_dispatch:
+
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  GHCR_USERNAME: ${{ secrets.GHCR_USERNAME }}
+    
+jobs:
+  build_and_push:
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install Apptainer
+        run: |
+          .github/scripts/install_apptainer.sh
+      - name: Check Apptainer installation
+        run: apptainer --version
+      - name: Install Wave CLI
+        run: |
+          .github/scripts/install_wave-cli.sh
+      - name: Check wave installation
+        run: wave --version
+      - name: Build images
+        run: |
+          .github/scripts/build_container_images_wave.sh
+      - name: Push images
+        run: |
+          echo ${GITHUB_TOKEN} | apptainer registry login -u ${GHCR_USERNAME} --password-stdin oras://ghcr.io
+          .github/scripts/push_container_images_wave.py --wave-jsons-dir wave_images --images-dir wave_images

nextflow.config

@@ -4,7 +4,7 @@ manifest {
     description = 'Alignment and variant calling pipeline'
     mainScript = 'main.nf'
     nextflowVersion = '>=20.01.0'
-    version = '0.1.13'
+    version = '0.1.14'
 }
 
 params {
@@ -72,6 +72,14 @@ profiles {
 	    conda.cacheDir = params.cache
 	}
     }
+
+    apptainer {
+        apptainer.enabled = true
+        process.container = "oras://ghcr.io/bccdc-phl/alignment-variants:775751a430ea765a"
+        if (params.cache){
+            apptainer.cacheDir = params.cache
+        }
+    }
 }
 
 process {
@@ -87,5 +95,6 @@ process {
     }
     withName: plot_coverage {
 	conda = "$baseDir/environments/plot_coverage.yml"
+    container = "oras://ghcr.io/bccdc-phl/alignment-variants-plot-coverage:e9de1da959e4e7e7"
     }
 }