Merge pull request #2033: Gracefully handle weighted sampling with no… #4606
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: | |
| - master | |
| # Routinely check that tests pass with new versions of dependencies. | |
| schedule: | |
| # Every day at 17:42 UTC / 9:42 Seattle (winter) / 10:42 Seattle (summer) | |
| - cron: "42 17 * * *" | |
| pull_request: | |
| workflow_dispatch: | |
| workflow_call: | |
| inputs: | |
| version: | |
| required: true | |
| type: string | |
| jobs: | |
| mypy: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - uses: actions/setup-python@v7 | |
| - run: pip install .[dev] | |
| - run: mypy | |
| pyright: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - run: npx pyright --stats | |
| pytest-cram: | |
| name: pytest-cram (${{ matrix.conda-args }} ${{ matrix.pip-args }}) | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| include: | |
| # All supported Python versions with the earliest compatible versions | |
| # of Biopython¹ and Numpy² | |
| # ¹ <https://github.com/biopython/biopython/blob/master/NEWS.rst> | |
| # ² <https://numpy.org/doc/stable/release> | |
| - { conda-args: 'python=3.10', pip-args: 'biopython==1.80 numpy==1.21.3' } | |
| - { conda-args: 'python=3.11', pip-args: 'biopython==1.80 numpy==1.23.4' } # numpy 1.23.2-3 did not declare support in conda | |
| - { conda-args: 'python=3.12', pip-args: 'biopython==1.82 numpy==1.26.0' } | |
| - { conda-args: 'python=3.13', pip-args: 'biopython==1.85 numpy==2.1.0' } | |
| - { conda-args: 'python=3.14', pip-args: 'biopython==1.86 numpy==2.3.2' } | |
| # Latest versions of Biopython and Numpy | |
| - { conda-args: 'python=3.14', pip-args: 'biopython numpy' } | |
| # Latest Biopython with Numpy v1 | |
| - { conda-args: 'python=3.12', pip-args: 'biopython numpy==1.*' } | |
| defaults: | |
| run: | |
| shell: bash -l {0} | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - name: Set cache key | |
| run: echo "DATE=$(date +'%Y-%m-%d')" >> "$GITHUB_ENV" | |
| # Create a Conda environment from dev_env.yml. | |
| # Python dependencies are left out, allowing us to test against dependencies | |
| # installed by pip, which may have slightly different versions compared to | |
| # Conda counterparts. | |
| - name: Install dependencies from Conda | |
| uses: mamba-org/setup-micromamba@v3 | |
| with: | |
| create-args: ${{ matrix.conda-args }} | |
| environment-file: dev_env.yml | |
| cache-environment: true | |
| cache-environment-key: ${{ env.DATE }} | |
| environment-name: augur | |
| # Install the local version of Augur and Python dependencies. | |
| - run: pip install .[dev] ${{ matrix.pip-args }} | |
| - run: pip list | |
| - run: conda info | |
| - run: conda list | |
| # Run with coverage on the earliest supported Python version | |
| # Ideally, coverage is run on all jobs and combined later.¹ It was reduced | |
| # down to a single to improve overall CI run time.² | |
| # ¹ <https://github.com/nextstrain/augur/pull/968> | |
| # ² <https://github.com/nextstrain/augur/pull/1861> | |
| - name: Set coverage environment variables | |
| if: contains(matrix.conda-args, 'python=3.10') | |
| run: | | |
| coverage_id="$(printf '%s' "${{ matrix.conda-args }}|${{ matrix.pip-args }}" | sha256sum | cut -d' ' -f1)" | |
| echo "COVERAGE_FILE=${{ github.workspace }}/.coverage@${coverage_id}" >> "$GITHUB_ENV" | |
| echo "COVERAGE_RCFILE=${{ github.workspace }}/.coveragerc" >> "$GITHUB_ENV" | |
| if python3 -c 'import sys; print(sys.version_info >= (3, 14))' | grep -q True; then | |
| echo "Using sysmon for coverage to reduce overhead" | |
| echo COVERAGE_CORE=sysmon >> "$GITHUB_ENV" | |
| else | |
| echo "Using default coverage collection method" | |
| fi | |
| - name: Run pytest | |
| run: | | |
| if [[ -n "${COVERAGE_FILE:-}" ]]; then | |
| echo "Running pytest with coverage enabled" | |
| pytest --cov=augur | |
| else | |
| echo "Running pytest without coverage" | |
| pytest --no-cov | |
| fi | |
| - name: Run cram tests | |
| run: | | |
| if [[ -n "${COVERAGE_FILE:-}" ]]; then | |
| echo "Running cram tests with coverage enabled" | |
| export AUGUR="coverage run -a ${{ github.workspace }}/bin/augur" | |
| else | |
| echo "Running cram tests without coverage" | |
| export AUGUR="${{ github.workspace }}/bin/augur" | |
| fi | |
| scripts/run-cram-parallel.py | |
| - name: Upload coverage | |
| if: env.COVERAGE_FILE | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: coverage | |
| include-hidden-files: true | |
| path: "${{ env.COVERAGE_FILE }}" | |
| # Replicating pathogen-repo-ci workflow because we decided not to support | |
| # local versions of Augur in the centralized workflow | |
| # <https://github.com/nextstrain/.github/issues/66> | |
| # This job is for pathogen repos _do_ follow standard pathogen repo structure | |
| # and new pathogens should be added here. | |
| pathogen-repo-ci: | |
| runs-on: ubuntu-latest | |
| continue-on-error: true | |
| strategy: | |
| matrix: | |
| pathogen: | |
| - dengue | |
| - lassa | |
| - measles | |
| - mpox | |
| - mumps | |
| - nipah | |
| - norovirus | |
| - oropouche | |
| - rabies | |
| - rubella | |
| - seasonal-cov | |
| - WNV | |
| - yellow-fever | |
| - zika | |
| name: pathogen-repo-ci (${{ matrix.pathogen }}) | |
| defaults: | |
| run: | |
| shell: bash -l {0} | |
| steps: | |
| - uses: actions/checkout@v7 | |
| with: | |
| path: ./augur | |
| - name: Set cache key | |
| run: echo "DATE=$(date +'%Y-%m-%d')" >> "$GITHUB_ENV" | |
| # Set up a Conda environment that replicates Nextstrain's Conda runtime. | |
| - name: Install nextstrain-base from Conda | |
| uses: mamba-org/setup-micromamba@v3 | |
| with: | |
| create-args: nextstrain-base | |
| condarc: | | |
| channels: | |
| - nextstrain | |
| - conda-forge | |
| - bioconda | |
| channel_priority: strict | |
| cache-environment: true | |
| cache-environment-key: ${{ env.DATE }} | |
| environment-name: augur | |
| # Replace the Conda Augur installation with the local version. | |
| - run: pip install ./augur | |
| - uses: actions/checkout@v7 | |
| with: | |
| repository: nextstrain/${{ matrix.pathogen }} | |
| path: ./pathogen-repo | |
| - name: Verify nextstrain-pathogen.yaml file | |
| run: > | |
| if [[ ! -f './pathogen-repo/nextstrain-pathogen.yaml' ]]; then | |
| echo "To use this workflow, there must be a 'nextstrain-pathogen.yaml' file present in the repository root"; | |
| exit 1; | |
| fi | |
| - name: Run ingest | |
| if: hashFiles('./pathogen-repo/ingest/Snakefile') && hashFiles('./pathogen-repo/ingest/build-configs/ci/config.yaml') | |
| id: ingest | |
| run: nextstrain build --ambient ./pathogen-repo/ingest --configfile build-configs/ci/config.yaml | |
| - name: Run phylogenetic | |
| if: hashFiles('./pathogen-repo/phylogenetic/Snakefile') && hashFiles('./pathogen-repo/phylogenetic/build-configs/ci/config.yaml') && !cancelled() | |
| id: phylogenetic | |
| run: nextstrain build --ambient ./pathogen-repo/phylogenetic --configfile build-configs/ci/config.yaml | |
| - name: Run nextclade | |
| if: hashFiles('./pathogen-repo/nextclade/Snakefile') && hashFiles('./pathogen-repo/nextclade/build-configs/ci/config.yaml') && !cancelled() | |
| id: nextclade | |
| run: nextstrain build --ambient ./pathogen-repo/nextclade --configfile build-configs/ci/config.yaml | |
| - if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: output-${{ matrix.pathogen }} | |
| if-no-files-found: ignore | |
| path: | | |
| ./pathogen-repo/ingest/.snakemake/log/ | |
| ./pathogen-repo/ingest/auspice/ | |
| ./pathogen-repo/ingest/benchmarks/ | |
| ./pathogen-repo/ingest/logs/ | |
| ./pathogen-repo/ingest/results/ | |
| ./pathogen-repo/phylogenetic/.snakemake/log/ | |
| ./pathogen-repo/phylogenetic/auspice/ | |
| ./pathogen-repo/phylogenetic/benchmarks/ | |
| ./pathogen-repo/phylogenetic/logs/ | |
| ./pathogen-repo/phylogenetic/results/ | |
| ./pathogen-repo/nextclade/.snakemake/log/ | |
| ./pathogen-repo/nextclade/auspice/ | |
| ./pathogen-repo/nextclade/benchmarks/ | |
| ./pathogen-repo/nextclade/logs/ | |
| ./pathogen-repo/nextclade/results/ | |
| - if: always() | |
| name: Verify a workflow ran | |
| env: | |
| # "outcome" is success/failure/cancelled/skipped _before_ | |
| # "continue-on-error" is applied to calculate "conclusion"; we no | |
| # longer use continue-on-error for these steps, but even so, | |
| # conceptually here what we want is outcome not conclusion. | |
| ingest: ${{ steps.ingest.outcome }} | |
| phylogenetic: ${{ steps.phylogenetic.outcome }} | |
| nextclade: ${{ steps.nextclade.outcome }} | |
| run: | | |
| # Show step outcomes in job logs… | |
| echo "ingest $ingest" | tee -a "$GITHUB_STEP_SUMMARY" | |
| echo "phylogenetic $phylogenetic" | tee -a "$GITHUB_STEP_SUMMARY" | |
| echo "nextclade $nextclade"| tee -a "$GITHUB_STEP_SUMMARY" | |
| # Assert status; we're good if we see at least one success and the | |
| # rest are success or skipped. | |
| [[ | |
| ($ingest == success || $phylogenetic == success || $nextclade == success) | |
| && ($ingest == success || $ingest == skipped) | |
| && ($phylogenetic == success || $phylogenetic == skipped) | |
| && ($nextclade == success || $nextclade == skipped) | |
| ]] | |
| # Replicating pathogen-repo-ci-v0 workflow because we decided not to support | |
| # local versions of Augur in the centralized workflow | |
| # <https://github.com/nextstrain/.github/issues/66> | |
| # This particular jobs is for pathogen repos that do not follow the standard | |
| # pathogen repo structure and is not expected to be updated. | |
| # Any new pathogen repos should be added to the job replicating the latest version | |
| # of the pathogen-repo-ci above. | |
| pathogen-repo-ci-v0: | |
| runs-on: ubuntu-latest | |
| continue-on-error: true | |
| env: | |
| repodata_use_zst: true | |
| strategy: | |
| matrix: | |
| include: | |
| - { pathogen: avian-flu, build-args: --snakefile segment-focused/Snakefile --configfile build-configs/ci/config.yaml } | |
| - { pathogen: ncov, build-args: all_regions -j 2 --profile nextstrain_profiles/nextstrain-ci } | |
| - { pathogen: rsv, build-args: --configfile config/ci.yaml } | |
| - { pathogen: seasonal-flu, build-args: --configfile profiles/ci/builds.yaml -p } | |
| - { pathogen: vcf-input-tutorial } | |
| name: pathogen-repo-ci-v0 (${{ matrix.pathogen }}) | |
| defaults: | |
| run: | |
| shell: bash -l {0} | |
| steps: | |
| - uses: actions/checkout@v7 | |
| with: | |
| path: ./augur | |
| - name: Set cache key | |
| run: echo "DATE=$(date +'%Y-%m-%d')" >> "$GITHUB_ENV" | |
| # Set up a Conda environment that replicates Nextstrain's Conda runtime. | |
| - name: Install nextstrain-base from Conda | |
| uses: mamba-org/setup-micromamba@v3 | |
| with: | |
| create-args: nextstrain-base | |
| condarc: | | |
| channels: | |
| - nextstrain | |
| - conda-forge | |
| - bioconda | |
| channel_priority: strict | |
| cache-environment: true | |
| cache-environment-key: ${{ env.DATE }} | |
| environment-name: augur | |
| # Replace the Conda Augur installation with the local version. | |
| - run: pip install ./augur | |
| - uses: actions/checkout@v7 | |
| with: | |
| repository: nextstrain/${{ matrix.pathogen }} | |
| path: ./pathogen-repo | |
| - name: Copy example data | |
| working-directory: ./pathogen-repo | |
| run: | | |
| if [[ -d example_data ]]; then | |
| mkdir -p data/ | |
| cp -r -v example_data/* data/ | |
| else | |
| echo No example data to copy. | |
| fi | |
| - run: nextstrain build --ambient ./pathogen-repo ${{ matrix.build-args }} | |
| - if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: output-${{ matrix.pathogen }} | |
| path: | | |
| ./pathogen-repo/auspice/ | |
| ./pathogen-repo/results/ | |
| ./pathogen-repo/benchmarks/ | |
| ./pathogen-repo/logs/ | |
| ./pathogen-repo/.snakemake/log/ | |
| codecov: | |
| if: github.repository == 'nextstrain/augur' | |
| needs: [pytest-cram] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - uses: actions/setup-python@v7 | |
| - run: pip install coverage | |
| - uses: actions/download-artifact@v8 | |
| with: | |
| name: coverage | |
| - run: coverage combine .coverage@* | |
| - run: coverage xml | |
| - uses: codecov/codecov-action@v7 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| fail_ci_if_error: false | |
| check-docs: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - uses: actions/setup-python@v7 | |
| with: | |
| # This should match the python version in .readthedocs.yml | |
| python-version: '3.11' | |
| - run: pip install .[dev] | |
| - run: ./devel/regenerate-developer-api-docs | |
| - name: Check for changes | |
| run: | | |
| if [[ -n $(git status --porcelain) ]]; then | |
| git add . | |
| git diff --staged >&2 | |
| echo "There are changes that affect the developer API docs. Please update: <https://github.com/nextstrain/augur/blob/-/docs/contribute/DEV_DOCS.md#regenerating-developer-api-docs>" >&2 | |
| echo "If there are changes to the Augur CLI, please manually adjust files under 'docs/usage/cli/'." >&2 | |
| exit 1 | |
| fi | |
| check-subsample-config-schema: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - uses: actions/setup-python@v7 | |
| - run: pip install .[dev] | |
| - run: ./devel/regenerate-subsample-schema | |
| - name: Check for changes | |
| run: | | |
| if [[ -n $(git status --porcelain) ]]; then | |
| git add . | |
| git diff --staged >&2 | |
| echo "There are changes that affect the augur subsample config schema. Please regenerate by running devel/regenerate-subsample-schema." >&2 | |
| exit 1 | |
| fi | |
| release: | |
| # Only run when called by the release workflow on the default branch | |
| if: github.workflow_ref == format('{0}/.github/workflows/release.yaml@refs/heads/{1}', github.repository, github.event.repository.default_branch) | |
| needs: [pytest-cram] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v7 | |
| with: | |
| # Fetch all branches and tags. | |
| fetch-depth: 0 | |
| - uses: actions/setup-python@v7 | |
| with: | |
| python-version: '3.10' | |
| - name: Set Nextstrain bot as git user | |
| run: | | |
| git config --global user.email "hello@nextstrain.org" | |
| git config --global user.name "Nextstrain bot" | |
| - run: python3 -m pip install --upgrade build twine | |
| - run: devel/release ${{ github.event.inputs.version }} | |
| - uses: actions/upload-artifact@v7 | |
| with: | |
| name: dist | |
| path: dist/ | |
| - run: git push origin master tag ${{ github.event.inputs.version }} | |
| - name: Publish to PyPI | |
| run: twine upload dist/* | |
| env: | |
| TWINE_USERNAME: __token__ | |
| TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} | |
| TWINE_REPOSITORY_URL: https://upload.pypi.org/legacy/ | |
| - name: Create GitHub Release | |
| run: ./devel/create-github-release "${{github.event.inputs.version }}" dist/* | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| rebuild-docker-image: | |
| needs: [release] | |
| runs-on: ubuntu-latest | |
| steps: | |
| # Delay for 10 minutes to allow the PyPI package to be available. | |
| # See https://github.com/nextstrain/docker-base/issues/128 | |
| - name: Sleep for 10 minutes | |
| run: sleep 600 | |
| shell: bash | |
| - run: gh workflow run ci.yml --repo nextstrain/docker-base | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GH_TOKEN_NEXTSTRAIN_BOT_WORKFLOW_DISPATCH }} |