diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..c6e3f9181 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,424 @@ +name: CI + +on: + push: + branches: + - master + pull_request: + +defaults: + run: + shell: bash + +permissions: + contents: read + +env: + DEPLOY: /tmp/lcdb-wf-test + LCDBWF_ENV: lcdb-wf-test + LCDBWF_ENV_R: lcdb-wf-test-r + LC_ALL: en_US.utf8 + LANG: en_US.utf8 + MINIFORGE_DIR: /home/runner/miniforge3 + + # Notes on caching: + # + # The user-settings.mkfg file included in the cache is created by sra-tools + # upon installation by conda. If it's not included, fastq-dump thinks it's + # mis-configured + # + # Named environments are kept in the /home/runner/miniforge3 dir, and + # restored upon cache hit. + # + # Notes on artifacts: + # + # Log files are uploaded as artifacts, even when a job fails, for easier + # debugging. They have a default retention time of 90 days. +jobs: + pytest: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + + - uses: actions/cache@v5 + with: + key: miniforge-${{ runner.os }}-${{ hashFiles('env.yml', 'env-r.yml') }} + path: | + /home/runner/miniforge3 + /home/runner/.ncbi/user-settings.mkfg + + - name: Install dependencies and conda envs + run: | + source ci/github-actions-helpers.sh + lcdbwf_install_system_deps + lcdbwf_ensure_envs + + - name: Run pytest and R test suites + run: | + source ci/github-actions-helpers.sh + lcdbwf_init_conda + conda activate "${LCDBWF_ENV}" + test/lcdb-wf-test unit_tests --pytest + test/lcdb-wf-test unit_tests --ensure-docs + test/lcdb-wf-test unit_tests --r-test + + chipseq: + runs-on: ubuntu-latest + needs: pytest + steps: + - uses: actions/checkout@v5 + + - uses: actions/cache@v5 + with: + key: miniforge-${{ runner.os }}-${{ hashFiles('env.yml', 'env-r.yml') }} + path: | + /home/runner/miniforge3 + /home/runner/.ncbi/user-settings.mkfg + + - name: Set up test environment + run: | + source ci/github-actions-helpers.sh + lcdbwf_install_system_deps + lcdbwf_ensure_envs + lcdbwf_deploy_and_get_data + + - name: Run chipseq workflow + run: | + source ci/github-actions-helpers.sh + lcdbwf_init_conda + conda activate "${LCDBWF_ENV}" + cd "${DEPLOY}/workflows/chipseq" + "${DEPLOY}/test/lcdb-wf-test" chipseq --run-workflow --use-conda -j2 -k -p + "${DEPLOY}/test/lcdb-wf-test" chipseq --trackhub + + - uses: actions/upload-artifact@v6 + if: always() + with: + name: chipseq-multiqc + path: /tmp/lcdb-wf-test/workflows/chipseq/data/chipseq_aggregation/multiqc.html + if-no-files-found: warn + + - uses: actions/upload-artifact@v6 + if: always() + with: + name: chipseq-logs + path: | + /tmp/lcdb-wf-test/workflows/chipseq/.snakemake/log/**/*.log + /tmp/lcdb-wf-test/workflows/chipseq/**/*.log + if-no-files-found: warn + + chipseq-misc: + runs-on: ubuntu-latest + needs: pytest + steps: + - uses: actions/checkout@v5 + + - uses: actions/cache@v5 + with: + key: miniforge-${{ runner.os }}-${{ hashFiles('env.yml', 'env-r.yml') }} + path: | + /home/runner/miniforge3 + /home/runner/.ncbi/user-settings.mkfg + + - name: Set up test environment + run: | + source ci/github-actions-helpers.sh + lcdbwf_install_system_deps + lcdbwf_ensure_envs + lcdbwf_deploy_and_get_data + + - name: Run chipseq misc workflow tests + run: | + source ci/github-actions-helpers.sh + lcdbwf_init_conda + conda activate "${LCDBWF_ENV}" + ORIG="$(lcdbwf_repo_root)" + cd "${DEPLOY}/workflows/chipseq" + ./run_test.sh --use-conda -j2 -k -p \ + --configfile "${ORIG}/test/test_configs/test_chipseq_regression.yaml" \ + --config sampletable="${ORIG}/test/test_configs/chipseq_one_run.tsv" \ + merged_bigwigs="{}" \ + --until bed_to_bigbed + + - uses: actions/upload-artifact@v6 + if: always() + with: + name: chipseq-misc-logs + path: | + /tmp/lcdb-wf-test/workflows/chipseq/.snakemake/log/**/*.log + /tmp/lcdb-wf-test/workflows/chipseq/**/*.log + if-no-files-found: warn + + rnaseq: + runs-on: ubuntu-latest + needs: pytest + steps: + - uses: actions/checkout@v5 + + - uses: actions/cache@v5 + with: + key: miniforge-${{ runner.os }}-${{ hashFiles('env.yml', 'env-r.yml') }} + path: | + /home/runner/miniforge3 + /home/runner/.ncbi/user-settings.mkfg + + - name: Set up test environment + run: | + source ci/github-actions-helpers.sh + lcdbwf_install_system_deps + lcdbwf_ensure_envs + lcdbwf_deploy_and_get_data + + - name: Run rnaseq workflow + run: | + source ci/github-actions-helpers.sh + lcdbwf_init_conda + conda activate "${LCDBWF_ENV}" + ORIG="$(lcdbwf_repo_root)" + cd "${DEPLOY}" + "${DEPLOY}/test/lcdb-wf-test" rnaseq --run-workflow -n + "${DEPLOY}/test/lcdb-wf-test" rnaseq --run-workflow --use-conda -j2 -k -p --orig "${ORIG}" + "${DEPLOY}/test/lcdb-wf-test" rnaseq --trackhub --orig "${ORIG}" + "${DEPLOY}/test/lcdb-wf-test" rnaseq --downstream + + tar -zcf /tmp/downstream.tar.gz workflows/rnaseq/downstream-test/ + cp workflows/rnaseq/downstream-test/rnaseq.html /tmp/rnaseq.html + cp workflows/rnaseq/downstream-test/functional-enrichment.html /tmp/functional-enrichment.html + cp workflows/rnaseq/downstream-test/gene-patterns.html /tmp/gene-patterns.html + cp workflows/rnaseq/data/rnaseq_aggregation/multiqc.html /tmp/multiqc.html + + - uses: actions/upload-artifact@v6 + if: always() + with: + name: rnaseq-artifacts + path: | + /tmp/downstream.tar.gz + /tmp/rnaseq.html + /tmp/multiqc.html + /tmp/functional-enrichment.html + /tmp/gene-patterns.html + if-no-files-found: warn + + - uses: actions/upload-artifact@v6 + if: always() + with: + name: rnaseq-logs + path: | + /tmp/lcdb-wf-test/workflows/rnaseq/.snakemake/log/**/*.log + /tmp/lcdb-wf-test/workflows/rnaseq/**/*.log + if-no-files-found: warn + + rnaseq-misc: + runs-on: ubuntu-latest + needs: pytest + steps: + - uses: actions/checkout@v5 + + - uses: actions/cache@v5 + with: + key: miniforge-${{ runner.os }}-${{ hashFiles('env.yml', 'env-r.yml') }} + path: | + /home/runner/miniforge3 + /home/runner/.ncbi/user-settings.mkfg + + - name: Set up test environment + run: | + source ci/github-actions-helpers.sh + lcdbwf_install_system_deps + lcdbwf_ensure_envs + lcdbwf_deploy_and_get_data + + - name: Run rnaseq misc tests + run: | + source ci/github-actions-helpers.sh + lcdbwf_init_conda + conda activate "${LCDBWF_ENV}" + ORIG="$(lcdbwf_repo_root)" + cd "${DEPLOY}" + "${DEPLOY}/test/lcdb-wf-test" rnaseq --run-workflow --sra-pe -k -p -j2 --use-conda --orig "${ORIG}" + "${DEPLOY}/test/lcdb-wf-test" rnaseq --run-workflow --sra-se -k -p -j2 --use-conda --orig "${ORIG}" + "${DEPLOY}/test/lcdb-wf-test" rnaseq --run-workflow --strandedness-pe -k -p -j2 --use-conda --orig "${ORIG}" + "${DEPLOY}/test/lcdb-wf-test" rnaseq --run-workflow --star-2pass -k -p -j2 --use-conda --orig "${ORIG}" + "${DEPLOY}/test/lcdb-wf-test" rnaseq --run-workflow --star-1pass -k -p -j2 --use-conda --orig "${ORIG}" + "${DEPLOY}/test/lcdb-wf-test" rnaseq --run-workflow --pe -k -p -j2 --use-conda --orig "${ORIG}" + + - uses: actions/upload-artifact@v6 + if: always() + with: + name: rnaseq-misc-logs + path: | + /tmp/lcdb-wf-test/workflows/rnaseq/.snakemake/log/**/*.log + /tmp/lcdb-wf-test/workflows/rnaseq/**/*.log + if-no-files-found: warn + + references: + runs-on: ubuntu-latest + needs: pytest + steps: + - uses: actions/checkout@v5 + + - uses: actions/cache@v5 + with: + key: miniforge-${{ runner.os }}-${{ hashFiles('env.yml', 'env-r.yml') }} + path: | + /home/runner/miniforge3 + /home/runner/.ncbi/user-settings.mkfg + + - name: Set up test environment + run: | + source ci/github-actions-helpers.sh + lcdbwf_install_system_deps + lcdbwf_ensure_envs + lcdbwf_deploy_and_get_data + + - name: Run references workflows + run: | + source ci/github-actions-helpers.sh + lcdbwf_init_conda + conda activate "${LCDBWF_ENV}" + ORIG="$(lcdbwf_repo_root)" + cd "${DEPLOY}" + "${DEPLOY}/test/lcdb-wf-test" references --run-workflow --configfile=config/config.yaml -j2 -p -k --orig "${ORIG}" + "${DEPLOY}/test/lcdb-wf-test" references --run-workflow --configfile="${ORIG}/test/test_configs/variant-calling.yaml" -j2 -p -k --orig "${ORIG}" + + - uses: actions/upload-artifact@v6 + if: always() + with: + name: references-logs + path: | + /tmp/lcdb-wf-test/workflows/references/.snakemake/log/**/*.log + /tmp/lcdb-wf-test/workflows/references/**/*.log + if-no-files-found: warn + + variantcalling: + runs-on: ubuntu-latest + needs: pytest + steps: + - uses: actions/checkout@v5 + + - uses: actions/cache@v5 + with: + key: miniforge-${{ runner.os }}-${{ hashFiles('env.yml', 'env-r.yml') }} + path: | + /home/runner/miniforge3 + /home/runner/.ncbi/user-settings.mkfg + + - name: Set up test environment + run: | + source ci/github-actions-helpers.sh + lcdbwf_install_system_deps + lcdbwf_ensure_envs + lcdbwf_deploy_and_get_data + + - name: Run variantcalling workflow + run: | + source ci/github-actions-helpers.sh + lcdbwf_init_conda + conda activate "${LCDBWF_ENV}" + cd "${DEPLOY}" + "${DEPLOY}/test/lcdb-wf-test" variantcalling --run-workflow -n + "${DEPLOY}/test/lcdb-wf-test" variantcalling --run-workflow --use-conda -j2 + + - uses: actions/upload-artifact@v6 + if: always() + with: + name: variantcalling-logs + path: | + /tmp/lcdb-wf-test/workflows/variant-calling/.snakemake/log/**/*.log + /tmp/lcdb-wf-test/workflows/variant-calling/**/*.log + if-no-files-found: warn + + + build-docs: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + + - uses: actions/cache@v5 + with: + key: miniforge-${{ runner.os }}-${{ hashFiles('env.yml', 'env-r.yml') }} + path: | + /home/runner/miniforge3 + /home/runner/.ncbi/user-settings.mkfg + + - name: Set up docs environment + run: | + source ci/github-actions-helpers.sh + lcdbwf_install_system_deps + lcdbwf_ensure_envs + lcdbwf_init_conda + conda install -n "${LCDBWF_ENV}" -y sphinx make yaml + + - name: Build docs + run: | + source ci/github-actions-helpers.sh + lcdbwf_init_conda + conda activate "${LCDBWF_ENV}" + make -C docs clean html SPHINXOPTS="-j2" + tar -zcf /tmp/docs.tar.gz -C docs/_build html + + - uses: actions/upload-artifact@v6 + if: always() + with: + name: docs-build + path: /tmp/docs.tar.gz + if-no-files-found: warn + + - uses: actions/upload-pages-artifact@v3 + if: always() + with: + path: docs/_build/html + + deploy-docs: + runs-on: ubuntu-latest + needs: build-docs + if: github.repository == 'lcdb/lcdb-wf' && github.event_name == 'push' && github.ref == 'refs/heads/master' + permissions: + contents: read + pages: write + id-token: write + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - uses: actions/configure-pages@v5 + + - id: deployment + uses: actions/deploy-pages@v4 + + report-env: + runs-on: ubuntu-latest + needs: + - chipseq + - chipseq-misc + - rnaseq + - rnaseq-misc + - references + - variantcalling + steps: + - uses: actions/checkout@v5 + + - uses: actions/cache@v5 + with: + key: miniforge-${{ runner.os }}-${{ hashFiles('env.yml', 'env-r.yml') }} + path: | + /home/runner/miniforge3 + /home/runner/.ncbi/user-settings.mkfg + + - name: Export environment definitions + run: | + source ci/github-actions-helpers.sh + lcdbwf_install_system_deps + lcdbwf_ensure_envs + lcdbwf_init_conda + conda env export -n "${LCDBWF_ENV}" > /tmp/env.yaml + conda env export -n "${LCDBWF_ENV_R}" > /tmp/env-r.yaml + + - uses: actions/upload-artifact@v6 + if: always() + with: + name: exported-envs + path: | + /tmp/env.yaml + /tmp/env-r.yaml + if-no-files-found: warn diff --git a/ci/github-actions-helpers.sh b/ci/github-actions-helpers.sh new file mode 100644 index 000000000..75837fcb6 --- /dev/null +++ b/ci/github-actions-helpers.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash + +set -euo pipefail + +export DEPLOY="${DEPLOY:-/tmp/lcdb-wf-test}" +export LCDBWF_ENV="${LCDBWF_ENV:-lcdb-wf-test}" +export LCDBWF_ENV_R="${LCDBWF_ENV_R:-lcdb-wf-test-r}" +export LC_ALL="${LC_ALL:-en_US.utf8}" +export LANG="${LANG:-en_US.utf8}" +export MINIFORGE_DIR="${MINIFORGE_DIR:-$HOME/miniforge3}" + +# Get top-level dir +lcdbwf_repo_root() { + git rev-parse --show-toplevel +} + +# Ubuntu packages to be installed each time +lcdbwf_install_system_deps() { + export DEBIAN_FRONTEND=noninteractive + sudo apt-get update + sudo apt-get install -y \ + curl \ + git \ + locales \ + locales-all \ + make \ + rsync \ + tree \ + wget \ + x11-utils + sudo rm -rf /var/lib/apt/lists/* + sudo localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 || true +} + +# Install if needed, activate, and add channels. This is for the main conda +# installation. +lcdbwf_init_conda() { + if [ ! -x "${MINIFORGE_DIR}/bin/conda" ]; then + curl -L -o /tmp/miniforge.sh \ + "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" + bash /tmp/miniforge.sh -b -p "${MINIFORGE_DIR}" + fi + + source "${MINIFORGE_DIR}/etc/profile.d/conda.sh" + conda activate + conda config --system --remove-key channels >/dev/null 2>&1 || true + conda config --system --add channels bioconda + conda config --system --add channels conda-forge + conda config --system --set channel_priority strict +} + +# Create envs if they don't already exist (e.g., from restoring a cache) +lcdbwf_ensure_envs() { + lcdbwf_init_conda + + if ! conda env list | awk 'NR > 2 {print $1}' | grep -qx "${LCDBWF_ENV}"; then + time conda env create -n "${LCDBWF_ENV}" --file env.yml + fi + + if ! conda env list | awk 'NR > 2 {print $1}' | grep -qx "${LCDBWF_ENV_R}"; then + time conda env create -n "${LCDBWF_ENV_R}" --file env-r.yml + fi +} + +# Clone locally to a temp dir, use the deploy script there to deploy to a new +# dir, copy over necessary test files (that are intentionally excluded by the +# normal deploy process) and download test data. +lcdbwf_deploy_and_get_data() { + local orig staging + + orig="$(lcdbwf_repo_root)" + staging=/tmp/lcdb-wf-source + + lcdbwf_init_conda + conda activate "${LCDBWF_ENV}" + conda info --envs + conda config --show + + rm -rf "${DEPLOY}" "${staging}" + git clone "${orig}" "${staging}" + git -C "${staging}" checkout --detach "${GITHUB_SHA:-HEAD}" + + python "${staging}/deploy.py" --flavor full --dest "${DEPLOY}" + + cp "${orig}/workflows/chipseq/run_test.sh" "${DEPLOY}/workflows/chipseq/run_test.sh" + cp "${orig}/workflows/rnaseq/run_test.sh" "${DEPLOY}/workflows/rnaseq/run_test.sh" + cp "${orig}/workflows/rnaseq/run_downstream_test.sh" "${DEPLOY}/workflows/rnaseq/run_downstream_test.sh" + cp "${orig}/workflows/references/run_test.sh" "${DEPLOY}/workflows/references/run_test.sh" + cp "${orig}/workflows/variant-calling/run_test.sh" "${DEPLOY}/workflows/variant-calling/run_test.sh" + + mkdir -p "${DEPLOY}/ci" "${DEPLOY}/test" + cp "${orig}/test/lcdb-wf-test" "${DEPLOY}/test/lcdb-wf-test" + cp "${orig}/test/workflow_test_params.yaml" "${DEPLOY}/test/workflow_test_params.yaml" + cp "${orig}/ci/get-data.py" "${DEPLOY}/ci/get-data.py" + cp "${orig}/ci/preprocessor.py" "${DEPLOY}/ci/preprocessor.py" + + ( + cd "${DEPLOY}" + test/lcdb-wf-test data --kind=all --verbose + ) +} diff --git a/wrappers/wrappers/fastq-dump/environment.yaml b/wrappers/wrappers/fastq-dump/environment.yaml index 6653b6cc8..956c331d1 100644 --- a/wrappers/wrappers/fastq-dump/environment.yaml +++ b/wrappers/wrappers/fastq-dump/environment.yaml @@ -2,4 +2,4 @@ channels: - conda-forge - bioconda dependencies: - - sra-tools>=3 + - sra-tools<3.4.1