From 4e3c6122b661c947e2cec1ba15703a936bcbcc78 Mon Sep 17 00:00:00 2001
From: Ryan Dale <115406+daler@users.noreply.github.com>
Date: Mon, 6 Apr 2026 14:56:49 -0400
Subject: [PATCH 01/11] initial attempt at migrating to github actions

---
 .github/workflows/ci.yml     | 362 +++++++++++++++++++++++++++++++++++
 ci/github-actions-helpers.sh |  93 +++++++++
 2 files changed, 455 insertions(+)
 create mode 100644 .github/workflows/ci.yml
 create mode 100644 ci/github-actions-helpers.sh

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
new file mode 100644
index 000000000..4e97fb992
--- /dev/null
+++ b/.github/workflows/ci.yml
@@ -0,0 +1,362 @@
+name: CI
+
+on:
+  push:
+  pull_request:
+
+defaults:
+  run:
+    shell: bash
+
+permissions:
+  contents: read
+
+env:
+  DEPLOY: /tmp/lcdb-wf-test
+  LCDBWF_ENV: lcdb-wf-test
+  LCDBWF_ENV_R: lcdb-wf-test-r
+  LC_ALL: en_US.utf8
+  LANG: en_US.utf8
+  MINIFORGE_DIR: /home/runner/miniforge3
+
+jobs:
+  pytest:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: actions/cache@v4
+        with:
+          key: miniforge-${{ runner.os }}-${{ hashFiles('env.yml', 'env-r.yml') }}
+          path: |
+            /home/runner/miniforge3
+            /home/runner/.ncbi/user-settings.mkfg
+
+      - name: Install dependencies and conda envs
+        run: |
+          source ci/github-actions-helpers.sh
+          lcdbwf_install_system_deps
+          lcdbwf_ensure_envs
+
+      - name: Run pytest and R test suites
+        run: |
+          source ci/github-actions-helpers.sh
+          lcdbwf_init_conda
+          conda activate "${LCDBWF_ENV}"
+          test/lcdb-wf-test unit_tests --pytest
+          test/lcdb-wf-test unit_tests --ensure-docs
+          test/lcdb-wf-test unit_tests --r-test
+
+  chipseq:
+    runs-on: ubuntu-latest
+    needs: pytest
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: actions/cache@v4
+        with:
+          key: miniforge-${{ runner.os }}-${{ hashFiles('env.yml', 'env-r.yml') }}
+          path: |
+            /home/runner/miniforge3
+            /home/runner/.ncbi/user-settings.mkfg
+
+      - name: Set up test environment
+        run: |
+          source ci/github-actions-helpers.sh
+          lcdbwf_install_system_deps
+          lcdbwf_ensure_envs
+          lcdbwf_get_data
+
+      - name: Run chipseq workflow
+        run: |
+          source ci/github-actions-helpers.sh
+          lcdbwf_init_conda
+          conda activate "${LCDBWF_ENV}"
+          cd "${DEPLOY}/workflows/chipseq"
+          "${DEPLOY}/test/lcdb-wf-test" chipseq --run-workflow --use-conda -j2 -k -p
+          "${DEPLOY}/test/lcdb-wf-test" chipseq --trackhub
+
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: chipseq-multiqc
+          path: /tmp/lcdb-wf-test/workflows/chipseq/data/chipseq_aggregation/multiqc.html
+          if-no-files-found: warn
+
+  chipseq-misc:
+    runs-on: ubuntu-latest
+    needs: pytest
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: actions/cache@v4
+        with:
+          key: miniforge-${{ runner.os }}-${{ hashFiles('env.yml', 'env-r.yml') }}
+          path: |
+            /home/runner/miniforge3
+            /home/runner/.ncbi/user-settings.mkfg
+
+      - name: Set up test environment
+        run: |
+          source ci/github-actions-helpers.sh
+          lcdbwf_install_system_deps
+          lcdbwf_ensure_envs
+          lcdbwf_get_data
+
+      - name: Run chipseq misc workflow tests
+        run: |
+          source ci/github-actions-helpers.sh
+          lcdbwf_init_conda
+          conda activate "${LCDBWF_ENV}"
+          ORIG="$(lcdbwf_repo_root)"
+          cd "${DEPLOY}/workflows/chipseq"
+          ./run_test.sh --use-conda -j2 -k -p \
+            --configfile "${ORIG}/test/test_configs/test_chipseq_regression.yaml" \
+            --config sampletable="${ORIG}/test/test_configs/chipseq_one_run.tsv" \
+            merged_bigwigs="{}" \
+            --until bed_to_bigbed
+
+  rnaseq:
+    runs-on: ubuntu-latest
+    needs: pytest
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: actions/cache@v4
+        with:
+          key: miniforge-${{ runner.os }}-${{ hashFiles('env.yml', 'env-r.yml') }}
+          path: |
+            /home/runner/miniforge3
+            /home/runner/.ncbi/user-settings.mkfg
+
+      - name: Set up test environment
+        run: |
+          source ci/github-actions-helpers.sh
+          lcdbwf_install_system_deps
+          lcdbwf_ensure_envs
+          lcdbwf_get_data
+
+      - name: Run rnaseq workflow
+        run: |
+          source ci/github-actions-helpers.sh
+          lcdbwf_init_conda
+          conda activate "${LCDBWF_ENV}"
+          ORIG="$(lcdbwf_repo_root)"
+          cd "${DEPLOY}"
+          "${DEPLOY}/test/lcdb-wf-test" rnaseq --run-workflow -n
+          "${DEPLOY}/test/lcdb-wf-test" rnaseq --run-workflow --use-conda -j2 -k -p --orig "${ORIG}"
+          "${DEPLOY}/test/lcdb-wf-test" rnaseq --trackhub --orig "${ORIG}"
+          "${DEPLOY}/test/lcdb-wf-test" rnaseq --downstream
+
+          tar -zcf /tmp/downstream.tar.gz workflows/rnaseq/downstream-test/
+          cp workflows/rnaseq/downstream-test/rnaseq.html /tmp/rnaseq.html
+          cp workflows/rnaseq/downstream-test/functional-enrichment.html /tmp/functional-enrichment.html
+          cp workflows/rnaseq/downstream-test/gene-patterns.html /tmp/gene-patterns.html
+          cp workflows/rnaseq/data/rnaseq_aggregation/multiqc.html /tmp/multiqc.html
+
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: rnaseq-artifacts
+          path: |
+            /tmp/downstream.tar.gz
+            /tmp/rnaseq.html
+            /tmp/multiqc.html
+            /tmp/functional-enrichment.html
+            /tmp/gene-patterns.html
+          if-no-files-found: warn
+
+  rnaseq-misc:
+    runs-on: ubuntu-latest
+    needs: pytest
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: actions/cache@v4
+        with:
+          key: miniforge-${{ runner.os }}-${{ hashFiles('env.yml', 'env-r.yml') }}
+          path: |
+            /home/runner/miniforge3
+            /home/runner/.ncbi/user-settings.mkfg
+
+      - name: Set up test environment
+        run: |
+          source ci/github-actions-helpers.sh
+          lcdbwf_install_system_deps
+          lcdbwf_ensure_envs
+          lcdbwf_get_data
+
+      - name: Run rnaseq misc tests
+        run: |
+          source ci/github-actions-helpers.sh
+          lcdbwf_init_conda
+          conda activate "${LCDBWF_ENV}"
+          ORIG="$(lcdbwf_repo_root)"
+          cd "${DEPLOY}"
+          "${DEPLOY}/test/lcdb-wf-test" rnaseq --run-workflow --sra-pe -k -p -j2 --use-conda --orig "${ORIG}"
+          "${DEPLOY}/test/lcdb-wf-test" rnaseq --run-workflow --sra-se -k -p -j2 --use-conda --orig "${ORIG}"
+          "${DEPLOY}/test/lcdb-wf-test" rnaseq --run-workflow --strandedness-pe -k -p -j2 --use-conda --orig "${ORIG}"
+          "${DEPLOY}/test/lcdb-wf-test" rnaseq --run-workflow --star-2pass -k -p -j2 --use-conda --orig "${ORIG}"
+          "${DEPLOY}/test/lcdb-wf-test" rnaseq --run-workflow --star-1pass -k -p -j2 --use-conda --orig "${ORIG}"
+          "${DEPLOY}/test/lcdb-wf-test" rnaseq --run-workflow --pe -k -p -j2 --use-conda --orig "${ORIG}"
+
+  references:
+    runs-on: ubuntu-latest
+    needs: pytest
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: actions/cache@v4
+        with:
+          key: miniforge-${{ runner.os }}-${{ hashFiles('env.yml', 'env-r.yml') }}
+          path: |
+            /home/runner/miniforge3
+            /home/runner/.ncbi/user-settings.mkfg
+
+      - name: Set up test environment
+        run: |
+          source ci/github-actions-helpers.sh
+          lcdbwf_install_system_deps
+          lcdbwf_ensure_envs
+          lcdbwf_get_data
+
+      - name: Run references workflows
+        run: |
+          source ci/github-actions-helpers.sh
+          lcdbwf_init_conda
+          conda activate "${LCDBWF_ENV}"
+          ORIG="$(lcdbwf_repo_root)"
+          cd "${DEPLOY}"
+          "${DEPLOY}/test/lcdb-wf-test" references --run-workflow --configfile=config/config.yaml -j2 -p -k --orig "${ORIG}"
+          "${DEPLOY}/test/lcdb-wf-test" references --run-workflow --configfile="${ORIG}/test/test_configs/variant-calling.yaml" -j2 -p -k --orig "${ORIG}"
+
+  variantcalling:
+    runs-on: ubuntu-latest
+    needs: pytest
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: actions/cache@v4
+        with:
+          key: miniforge-${{ runner.os }}-${{ hashFiles('env.yml', 'env-r.yml') }}
+          path: |
+            /home/runner/miniforge3
+            /home/runner/.ncbi/user-settings.mkfg
+
+      - name: Set up test environment
+        run: |
+          source ci/github-actions-helpers.sh
+          lcdbwf_install_system_deps
+          lcdbwf_ensure_envs
+          lcdbwf_get_data
+
+      - name: Run variantcalling workflow
+        run: |
+          source ci/github-actions-helpers.sh
+          lcdbwf_init_conda
+          conda activate "${LCDBWF_ENV}"
+          cd "${DEPLOY}"
+          "${DEPLOY}/test/lcdb-wf-test" variantcalling --run-workflow -n
+          "${DEPLOY}/test/lcdb-wf-test" variantcalling --run-workflow --use-conda -j2
+          tar -zcf /tmp/variantcalling.tar.gz workflows/variant-calling/results
+
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: variantcalling-results
+          path: /tmp/variantcalling.tar.gz
+          if-no-files-found: warn
+
+  build-docs:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: actions/cache@v4
+        with:
+          key: miniforge-${{ runner.os }}-${{ hashFiles('env.yml', 'env-r.yml') }}
+          path: |
+            /home/runner/miniforge3
+            /home/runner/.ncbi/user-settings.mkfg
+
+      - name: Set up docs environment
+        run: |
+          source ci/github-actions-helpers.sh
+          lcdbwf_install_system_deps
+          lcdbwf_ensure_envs
+          lcdbwf_init_conda
+          conda install -n "${LCDBWF_ENV}" -y sphinx make yaml
+
+      - name: Build docs
+        run: |
+          source ci/github-actions-helpers.sh
+          lcdbwf_init_conda
+          conda activate "${LCDBWF_ENV}"
+          make -C docs clean html SPHINXOPTS="-j2"
+          tar -zcf /tmp/docs.tar.gz -C docs/_build html
+
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: docs-build
+          path: /tmp/docs.tar.gz
+          if-no-files-found: warn
+
+      - uses: actions/upload-pages-artifact@v3
+        if: always()
+        with:
+          path: docs/_build/html
+
+  deploy-docs:
+    runs-on: ubuntu-latest
+    needs: build-docs
+    if: github.repository == 'lcdb/lcdb-wf' && github.event_name == 'push' && github.ref == 'refs/heads/master'
+    permissions:
+      contents: read
+      pages: write
+      id-token: write
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    steps:
+      - uses: actions/configure-pages@v5
+
+      - id: deployment
+        uses: actions/deploy-pages@v4
+
+  report-env:
+    runs-on: ubuntu-latest
+    needs:
+      - chipseq
+      - chipseq-misc
+      - rnaseq
+      - rnaseq-misc
+      - references
+      - variantcalling
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: actions/cache@v4
+        with:
+          key: miniforge-${{ runner.os }}-${{ hashFiles('env.yml', 'env-r.yml') }}
+          path: |
+            /home/runner/miniforge3
+            /home/runner/.ncbi/user-settings.mkfg
+
+      - name: Export environment definitions
+        run: |
+          source ci/github-actions-helpers.sh
+          lcdbwf_install_system_deps
+          lcdbwf_ensure_envs
+          lcdbwf_init_conda
+          conda env export -n "${LCDBWF_ENV}" > /tmp/env.yaml
+          conda env export -n "${LCDBWF_ENV_R}" > /tmp/env-r.yaml
+
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: exported-envs
+          path: |
+            /tmp/env.yaml
+            /tmp/env-r.yaml
+          if-no-files-found: warn
diff --git a/ci/github-actions-helpers.sh b/ci/github-actions-helpers.sh
new file mode 100644
index 000000000..a7cb00ea6
--- /dev/null
+++ b/ci/github-actions-helpers.sh
@@ -0,0 +1,93 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+export DEPLOY="${DEPLOY:-/tmp/lcdb-wf-test}"
+export LCDBWF_ENV="${LCDBWF_ENV:-lcdb-wf-test}"
+export LCDBWF_ENV_R="${LCDBWF_ENV_R:-lcdb-wf-test-r}"
+export LC_ALL="${LC_ALL:-en_US.utf8}"
+export LANG="${LANG:-en_US.utf8}"
+export MINIFORGE_DIR="${MINIFORGE_DIR:-$HOME/miniforge3}"
+
+lcdbwf_repo_root() {
+  git rev-parse --show-toplevel
+}
+
+lcdbwf_install_system_deps() {
+  export DEBIAN_FRONTEND=noninteractive
+  sudo apt-get update
+  sudo apt-get install -y \
+    curl \
+    git \
+    locales \
+    locales-all \
+    make \
+    rsync \
+    tree \
+    wget \
+    x11-utils
+  sudo rm -rf /var/lib/apt/lists/*
+  sudo localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 || true
+}
+
+lcdbwf_init_conda() {
+  if [ ! -x "${MINIFORGE_DIR}/bin/conda" ]; then
+    curl -L -o /tmp/miniforge.sh \
+      "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh"
+    bash /tmp/miniforge.sh -b -p "${MINIFORGE_DIR}"
+  fi
+
+  source "${MINIFORGE_DIR}/etc/profile.d/conda.sh"
+  conda activate
+  conda config --system --remove-key channels >/dev/null 2>&1 || true
+  conda config --system --add channels bioconda
+  conda config --system --add channels conda-forge
+  conda config --system --set channel_priority strict
+}
+
+lcdbwf_ensure_envs() {
+  lcdbwf_init_conda
+
+  if ! conda env list | awk 'NR > 2 {print $1}' | grep -qx "${LCDBWF_ENV}"; then
+    time conda env create -n "${LCDBWF_ENV}" --file env.yml
+  fi
+
+  if ! conda env list | awk 'NR > 2 {print $1}' | grep -qx "${LCDBWF_ENV_R}"; then
+    time conda env create -n "${LCDBWF_ENV_R}" --file env-r.yml
+  fi
+}
+
+lcdbwf_get_data() {
+  local orig staging
+
+  orig="$(lcdbwf_repo_root)"
+  staging=/tmp/lcdb-wf-source
+
+  lcdbwf_init_conda
+  conda activate "${LCDBWF_ENV}"
+  conda info --envs
+  conda config --show
+
+  rm -rf "${DEPLOY}" "${staging}"
+  git clone "${orig}" "${staging}"
+  git -C "${staging}" checkout --detach "${GITHUB_SHA:-HEAD}"
+
+  python "${staging}/deploy.py" --flavor full --dest "${DEPLOY}"
+
+  cp "${orig}/workflows/chipseq/run_test.sh" "${DEPLOY}/workflows/chipseq/run_test.sh"
+  cp "${orig}/workflows/rnaseq/run_test.sh" "${DEPLOY}/workflows/rnaseq/run_test.sh"
+  cp "${orig}/workflows/rnaseq/run_downstream_test.sh" "${DEPLOY}/workflows/rnaseq/run_downstream_test.sh"
+  cp "${orig}/workflows/references/run_test.sh" "${DEPLOY}/workflows/references/run_test.sh"
+  cp "${orig}/workflows/variant-calling/run_test.sh" "${DEPLOY}/workflows/variant-calling/run_test.sh"
+
+  mkdir -p "${DEPLOY}/ci" "${DEPLOY}/test"
+  cp "${orig}/test/lcdb-wf-test" "${DEPLOY}/test/lcdb-wf-test"
+  cp "${orig}/test/workflow_test_params.yaml" "${DEPLOY}/test/workflow_test_params.yaml"
+  cp "${orig}/ci/get-data.py" "${DEPLOY}/ci/get-data.py"
+  cp "${orig}/ci/preprocessor.py" "${DEPLOY}/ci/preprocessor.py"
+
+  (
+    cd "${DEPLOY}"
+    test/lcdb-wf-test data --kind=all --verbose
+  )
+}

From 8a577a966f04da9bd2b12b804cae1ea59a1d8270 Mon Sep 17 00:00:00 2001
From: Ryan Dale <115406+daler@users.noreply.github.com>
Date: Mon, 6 Apr 2026 15:20:07 -0400
Subject: [PATCH 02/11] always upload log artifacts (useful for debugging)

---
 .github/workflows/ci.yml | 54 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 54 insertions(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 4e97fb992..1ec99760c 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -83,6 +83,15 @@ jobs:
           path: /tmp/lcdb-wf-test/workflows/chipseq/data/chipseq_aggregation/multiqc.html
           if-no-files-found: warn
 
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: chipseq-logs
+          path: |
+            /tmp/lcdb-wf-test/workflows/chipseq/.snakemake/log/**/*.log
+            /tmp/lcdb-wf-test/workflows/chipseq/**/*.log
+          if-no-files-found: warn
+
   chipseq-misc:
     runs-on: ubuntu-latest
     needs: pytest
@@ -116,6 +125,15 @@ jobs:
             merged_bigwigs="{}" \
             --until bed_to_bigbed
 
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: chipseq-misc-logs
+          path: |
+            /tmp/lcdb-wf-test/workflows/chipseq/.snakemake/log/**/*.log
+            /tmp/lcdb-wf-test/workflows/chipseq/**/*.log
+          if-no-files-found: warn
+
   rnaseq:
     runs-on: ubuntu-latest
     needs: pytest
@@ -166,6 +184,15 @@ jobs:
             /tmp/gene-patterns.html
           if-no-files-found: warn
 
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: rnaseq-logs
+          path: |
+            /tmp/lcdb-wf-test/workflows/rnaseq/.snakemake/log/**/*.log
+            /tmp/lcdb-wf-test/workflows/rnaseq/**/*.log
+          if-no-files-found: warn
+
   rnaseq-misc:
     runs-on: ubuntu-latest
     needs: pytest
@@ -200,6 +227,15 @@ jobs:
           "${DEPLOY}/test/lcdb-wf-test" rnaseq --run-workflow --star-1pass -k -p -j2 --use-conda --orig "${ORIG}"
           "${DEPLOY}/test/lcdb-wf-test" rnaseq --run-workflow --pe -k -p -j2 --use-conda --orig "${ORIG}"
 
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: rnaseq-misc-logs
+          path: |
+            /tmp/lcdb-wf-test/workflows/rnaseq/.snakemake/log/**/*.log
+            /tmp/lcdb-wf-test/workflows/rnaseq/**/*.log
+          if-no-files-found: warn
+
   references:
     runs-on: ubuntu-latest
     needs: pytest
@@ -230,6 +266,15 @@ jobs:
           "${DEPLOY}/test/lcdb-wf-test" references --run-workflow --configfile=config/config.yaml -j2 -p -k --orig "${ORIG}"
           "${DEPLOY}/test/lcdb-wf-test" references --run-workflow --configfile="${ORIG}/test/test_configs/variant-calling.yaml" -j2 -p -k --orig "${ORIG}"
 
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: references-logs
+          path: |
+            /tmp/lcdb-wf-test/workflows/references/.snakemake/log/**/*.log
+            /tmp/lcdb-wf-test/workflows/references/**/*.log
+          if-no-files-found: warn
+
   variantcalling:
     runs-on: ubuntu-latest
     needs: pytest
@@ -260,6 +305,15 @@ jobs:
           "${DEPLOY}/test/lcdb-wf-test" variantcalling --run-workflow --use-conda -j2
           tar -zcf /tmp/variantcalling.tar.gz workflows/variant-calling/results
 
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: variantcalling-logs
+          path: |
+            /tmp/lcdb-wf-test/workflows/variant-calling/.snakemake/log/**/*.log
+            /tmp/lcdb-wf-test/workflows/variant-calling/**/*.log
+          if-no-files-found: warn
+
       - uses: actions/upload-artifact@v4
         if: always()
         with:

From f39fd70c901a3d5bd08eeeedab24ce4bf2fb53f0 Mon Sep 17 00:00:00 2001
From: Ryan Dale <115406+daler@users.noreply.github.com>
Date: Mon, 6 Apr 2026 15:28:54 -0400
Subject: [PATCH 03/11] try pinning sra tools since it's segfaulting otherwise

---
 wrappers/wrappers/fastq-dump/environment.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/wrappers/wrappers/fastq-dump/environment.yaml b/wrappers/wrappers/fastq-dump/environment.yaml
index 6653b6cc8..3af218091 100644
--- a/wrappers/wrappers/fastq-dump/environment.yaml
+++ b/wrappers/wrappers/fastq-dump/environment.yaml
@@ -2,4 +2,4 @@ channels:
   - conda-forge
   - bioconda
 dependencies:
-  - sra-tools>=3
+  - sra-tools=2.9.6

From 3f7492ef847740d0fa94188512017addb9df50ba Mon Sep 17 00:00:00 2001
From: Ryan Dale <115406+daler@users.noreply.github.com>
Date: Tue, 7 Apr 2026 09:27:38 -0400
Subject: [PATCH 04/11] sra-tools 3.4.1 segfaults, pin to lower

---
 wrappers/wrappers/fastq-dump/environment.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/wrappers/wrappers/fastq-dump/environment.yaml b/wrappers/wrappers/fastq-dump/environment.yaml
index 3af218091..956c331d1 100644
--- a/wrappers/wrappers/fastq-dump/environment.yaml
+++ b/wrappers/wrappers/fastq-dump/environment.yaml
@@ -2,4 +2,4 @@ channels:
   - conda-forge
   - bioconda
 dependencies:
-  - sra-tools=2.9.6
+  - sra-tools<3.4.1

From 7e20452f96c8fb4e8e820e302c42487e7f46ad4f Mon Sep 17 00:00:00 2001
From: Ryan Dale <115406+daler@users.noreply.github.com>
Date: Tue, 7 Apr 2026 09:51:11 -0400
Subject: [PATCH 05/11] use actions/cache@v5 to avoide node.js v20 warnings

---
 .github/workflows/ci.yml | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 1ec99760c..6ac3110ef 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -25,7 +25,7 @@ jobs:
     steps:
       - uses: actions/checkout@v5
 
-      - uses: actions/cache@v4
+      - uses: actions/cache@v5
         with:
           key: miniforge-${{ runner.os }}-${{ hashFiles('env.yml', 'env-r.yml') }}
           path: |
@@ -53,7 +53,7 @@ jobs:
     steps:
       - uses: actions/checkout@v5
 
-      - uses: actions/cache@v4
+      - uses: actions/cache@v5
         with:
           key: miniforge-${{ runner.os }}-${{ hashFiles('env.yml', 'env-r.yml') }}
           path: |
@@ -98,7 +98,7 @@ jobs:
     steps:
       - uses: actions/checkout@v5
 
-      - uses: actions/cache@v4
+      - uses: actions/cache@v5
         with:
           key: miniforge-${{ runner.os }}-${{ hashFiles('env.yml', 'env-r.yml') }}
           path: |
@@ -140,7 +140,7 @@ jobs:
     steps:
       - uses: actions/checkout@v5
 
-      - uses: actions/cache@v4
+      - uses: actions/cache@v5
         with:
           key: miniforge-${{ runner.os }}-${{ hashFiles('env.yml', 'env-r.yml') }}
           path: |
@@ -199,7 +199,7 @@ jobs:
     steps:
       - uses: actions/checkout@v5
 
-      - uses: actions/cache@v4
+      - uses: actions/cache@v5
         with:
           key: miniforge-${{ runner.os }}-${{ hashFiles('env.yml', 'env-r.yml') }}
           path: |
@@ -242,7 +242,7 @@ jobs:
     steps:
       - uses: actions/checkout@v5
 
-      - uses: actions/cache@v4
+      - uses: actions/cache@v5
         with:
           key: miniforge-${{ runner.os }}-${{ hashFiles('env.yml', 'env-r.yml') }}
           path: |
@@ -281,7 +281,7 @@ jobs:
     steps:
       - uses: actions/checkout@v5
 
-      - uses: actions/cache@v4
+      - uses: actions/cache@v5
         with:
           key: miniforge-${{ runner.os }}-${{ hashFiles('env.yml', 'env-r.yml') }}
           path: |
@@ -326,7 +326,7 @@ jobs:
     steps:
       - uses: actions/checkout@v5
 
-      - uses: actions/cache@v4
+      - uses: actions/cache@v5
         with:
           key: miniforge-${{ runner.os }}-${{ hashFiles('env.yml', 'env-r.yml') }}
           path: |
@@ -390,7 +390,7 @@ jobs:
     steps:
       - uses: actions/checkout@v5
 
-      - uses: actions/cache@v4
+      - uses: actions/cache@v5
         with:
           key: miniforge-${{ runner.os }}-${{ hashFiles('env.yml', 'env-r.yml') }}
           path: |

From 9cb7463b47d09cfb5509bb6d15b10da923e46976 Mon Sep 17 00:00:00 2001
From: Ryan Dale <115406+daler@users.noreply.github.com>
Date: Tue, 7 Apr 2026 10:03:17 -0400
Subject: [PATCH 06/11] add comments to GHA helper functions

---
 ci/github-actions-helpers.sh | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/ci/github-actions-helpers.sh b/ci/github-actions-helpers.sh
index a7cb00ea6..75837fcb6 100644
--- a/ci/github-actions-helpers.sh
+++ b/ci/github-actions-helpers.sh
@@ -9,10 +9,12 @@ export LC_ALL="${LC_ALL:-en_US.utf8}"
 export LANG="${LANG:-en_US.utf8}"
 export MINIFORGE_DIR="${MINIFORGE_DIR:-$HOME/miniforge3}"
 
+# Get top-level dir
 lcdbwf_repo_root() {
   git rev-parse --show-toplevel
 }
 
+# Ubuntu packages to be installed each time
 lcdbwf_install_system_deps() {
   export DEBIAN_FRONTEND=noninteractive
   sudo apt-get update
@@ -30,6 +32,8 @@ lcdbwf_install_system_deps() {
   sudo localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 || true
 }
 
+# Install if needed, activate, and add channels. This is for the main conda
+# installation.
 lcdbwf_init_conda() {
   if [ ! -x "${MINIFORGE_DIR}/bin/conda" ]; then
     curl -L -o /tmp/miniforge.sh \
@@ -45,6 +49,7 @@ lcdbwf_init_conda() {
   conda config --system --set channel_priority strict
 }
 
+# Create envs if they don't already exist (e.g., from restoring a cache)
 lcdbwf_ensure_envs() {
   lcdbwf_init_conda
 
@@ -57,7 +62,10 @@ lcdbwf_ensure_envs() {
   fi
 }
 
-lcdbwf_get_data() {
+# Clone locally to a temp dir, use the deploy script there to deploy to a new
+# dir, copy over necessary test files (that are intentionally excluded by the
+# normal deploy process) and download test data.
+lcdbwf_deploy_and_get_data() {
   local orig staging
 
   orig="$(lcdbwf_repo_root)"

From a8b3e9b4e8b66244e6c78602f625f07159512f8a Mon Sep 17 00:00:00 2001
From: Ryan Dale <115406+daler@users.noreply.github.com>
Date: Tue, 7 Apr 2026 10:03:43 -0400
Subject: [PATCH 07/11] rename function

---
 .github/workflows/ci.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 6ac3110ef..538a52332 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -65,7 +65,7 @@ jobs:
           source ci/github-actions-helpers.sh
           lcdbwf_install_system_deps
           lcdbwf_ensure_envs
-          lcdbwf_get_data
+          lcdbwf_deploy_and_get_data
 
       - name: Run chipseq workflow
         run: |
@@ -110,7 +110,7 @@ jobs:
           source ci/github-actions-helpers.sh
           lcdbwf_install_system_deps
           lcdbwf_ensure_envs
-          lcdbwf_get_data
+          lcdbwf_deploy_and_get_data
 
       - name: Run chipseq misc workflow tests
         run: |
@@ -152,7 +152,7 @@ jobs:
           source ci/github-actions-helpers.sh
           lcdbwf_install_system_deps
           lcdbwf_ensure_envs
-          lcdbwf_get_data
+          lcdbwf_deploy_and_get_data
 
       - name: Run rnaseq workflow
         run: |
@@ -211,7 +211,7 @@ jobs:
           source ci/github-actions-helpers.sh
           lcdbwf_install_system_deps
           lcdbwf_ensure_envs
-          lcdbwf_get_data
+          lcdbwf_deploy_and_get_data
 
       - name: Run rnaseq misc tests
         run: |
@@ -254,7 +254,7 @@ jobs:
           source ci/github-actions-helpers.sh
           lcdbwf_install_system_deps
           lcdbwf_ensure_envs
-          lcdbwf_get_data
+          lcdbwf_deploy_and_get_data
 
       - name: Run references workflows
         run: |
@@ -293,7 +293,7 @@ jobs:
           source ci/github-actions-helpers.sh
           lcdbwf_install_system_deps
           lcdbwf_ensure_envs
-          lcdbwf_get_data
+          lcdbwf_deploy_and_get_data
 
       - name: Run variantcalling workflow
         run: |

From ca4f467df5643c1d07a6e04780705f19b1e09c9e Mon Sep 17 00:00:00 2001
From: Ryan Dale <115406+daler@users.noreply.github.com>
Date: Tue, 7 Apr 2026 10:04:08 -0400
Subject: [PATCH 08/11] only run push tests on master; run on PR

---
 .github/workflows/ci.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 538a52332..a329d134b 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -2,6 +2,8 @@ name: CI
 
 on:
   push:
+    branches:
+      - master
   pull_request:
 
 defaults:

From 300ac78e9b97c86fae180eb1d61b6f60c27039ad Mon Sep 17 00:00:00 2001
From: Ryan Dale <115406+daler@users.noreply.github.com>
Date: Tue, 7 Apr 2026 10:27:19 -0400
Subject: [PATCH 09/11] bump upload-artifact action

---
 .github/workflows/ci.yml | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a329d134b..82e3de745 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -78,14 +78,14 @@ jobs:
           "${DEPLOY}/test/lcdb-wf-test" chipseq --run-workflow --use-conda -j2 -k -p
           "${DEPLOY}/test/lcdb-wf-test" chipseq --trackhub
 
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v6
         if: always()
         with:
           name: chipseq-multiqc
           path: /tmp/lcdb-wf-test/workflows/chipseq/data/chipseq_aggregation/multiqc.html
           if-no-files-found: warn
 
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v6
         if: always()
         with:
           name: chipseq-logs
@@ -127,7 +127,7 @@ jobs:
             merged_bigwigs="{}" \
             --until bed_to_bigbed
 
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v6
         if: always()
         with:
           name: chipseq-misc-logs
@@ -174,7 +174,7 @@ jobs:
           cp workflows/rnaseq/downstream-test/gene-patterns.html /tmp/gene-patterns.html
           cp workflows/rnaseq/data/rnaseq_aggregation/multiqc.html /tmp/multiqc.html
 
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v6
         if: always()
         with:
           name: rnaseq-artifacts
@@ -186,7 +186,7 @@ jobs:
             /tmp/gene-patterns.html
           if-no-files-found: warn
 
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v6
         if: always()
         with:
           name: rnaseq-logs
@@ -229,7 +229,7 @@ jobs:
           "${DEPLOY}/test/lcdb-wf-test" rnaseq --run-workflow --star-1pass -k -p -j2 --use-conda --orig "${ORIG}"
           "${DEPLOY}/test/lcdb-wf-test" rnaseq --run-workflow --pe -k -p -j2 --use-conda --orig "${ORIG}"
 
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v6
         if: always()
         with:
           name: rnaseq-misc-logs
@@ -268,7 +268,7 @@ jobs:
           "${DEPLOY}/test/lcdb-wf-test" references --run-workflow --configfile=config/config.yaml -j2 -p -k --orig "${ORIG}"
           "${DEPLOY}/test/lcdb-wf-test" references --run-workflow --configfile="${ORIG}/test/test_configs/variant-calling.yaml" -j2 -p -k --orig "${ORIG}"
 
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v6
         if: always()
         with:
           name: references-logs
@@ -307,7 +307,7 @@ jobs:
           "${DEPLOY}/test/lcdb-wf-test" variantcalling --run-workflow --use-conda -j2
           tar -zcf /tmp/variantcalling.tar.gz workflows/variant-calling/results
 
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v6
         if: always()
         with:
           name: variantcalling-logs
@@ -316,7 +316,7 @@ jobs:
             /tmp/lcdb-wf-test/workflows/variant-calling/**/*.log
           if-no-files-found: warn
 
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v6
         if: always()
         with:
           name: variantcalling-results
@@ -351,7 +351,7 @@ jobs:
           make -C docs clean html SPHINXOPTS="-j2"
           tar -zcf /tmp/docs.tar.gz -C docs/_build html
 
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v6
         if: always()
         with:
           name: docs-build
@@ -408,7 +408,7 @@ jobs:
           conda env export -n "${LCDBWF_ENV}" > /tmp/env.yaml
           conda env export -n "${LCDBWF_ENV_R}" > /tmp/env-r.yaml
 
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v6
         if: always()
         with:
           name: exported-envs

From 6a88fd7cdf962728074cd15e5e2e20dd06d40473 Mon Sep 17 00:00:00 2001
From: Ryan Dale <115406+daler@users.noreply.github.com>
Date: Tue, 7 Apr 2026 11:09:14 -0400
Subject: [PATCH 10/11] don't upload vc results as artifact

---
 .github/workflows/ci.yml | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 82e3de745..17cf3d0b3 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -305,7 +305,6 @@ jobs:
           cd "${DEPLOY}"
           "${DEPLOY}/test/lcdb-wf-test" variantcalling --run-workflow -n
           "${DEPLOY}/test/lcdb-wf-test" variantcalling --run-workflow --use-conda -j2
-          tar -zcf /tmp/variantcalling.tar.gz workflows/variant-calling/results
 
       - uses: actions/upload-artifact@v6
         if: always()
@@ -316,12 +315,6 @@ jobs:
             /tmp/lcdb-wf-test/workflows/variant-calling/**/*.log
           if-no-files-found: warn
 
-      - uses: actions/upload-artifact@v6
-        if: always()
-        with:
-          name: variantcalling-results
-          path: /tmp/variantcalling.tar.gz
-          if-no-files-found: warn
 
   build-docs:
     runs-on: ubuntu-latest

From a06d6c4f21a07146eb39a9d9b10e192c662c3d97 Mon Sep 17 00:00:00 2001
From: Ryan Dale <115406+daler@users.noreply.github.com>
Date: Tue, 7 Apr 2026 11:09:36 -0400
Subject: [PATCH 11/11] notes on caching and artifacts

---
 .github/workflows/ci.yml | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 17cf3d0b3..c6e3f9181 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -21,6 +21,19 @@ env:
   LANG: en_US.utf8
   MINIFORGE_DIR: /home/runner/miniforge3
 
+  # Notes on caching:
+  #
+  # The user-settings.mkfg file included in the cache is created by sra-tools
+  # upon installation by conda. If it's not included, fastq-dump thinks it's
+  # mis-configured
+  #
+  # Named environments are kept in the /home/runner/miniforge3 dir, and
+  # restored upon cache hit.
+  #
+  # Notes on artifacts:
+  #
+  # Log files are uploaded as artifacts, even when a job fails, for easier
+  # debugging. They have a default retention time of 90 days.
 jobs:
   pytest:
     runs-on: ubuntu-latest