Merge branch 'main' into release_hub

Author: canergen

.github/workflows/release.yml

@@ -1,32 +1,163 @@
 name: Release
 
+# based off of https://github.com/astral-sh/ruff/blob/main/.github/workflows/release.yaml
 on:
-  push:
-    tags:
-      - "*.*.*"
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: "The version to tag (without the leading 'v'). If omitted, will initiate a dry run (no uploads)."
+        default: ""
+        type: string
+      branch:
+        description: "The branch from which to release."
+        type: string
+      prerelease:
+        description: "Whether the release is a pre-release."
+        default: false
+        type: boolean
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 jobs:
-  release:
-    name: Release
+  validate-tag:
+    name: Validate tag
     runs-on: ubuntu-latest
+    # If you don't set an input tag, it's a dry run (no uploads).
+    if: ${{ inputs.tag }}
     steps:
-      # will use ref/SHA that triggered it
-      - name: Checkout code
-        uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.branch }}
+
+      - name: Check tag consistency
+        run: |
+          # Switch to the commit we want to release
+          git checkout ${{ inputs.branch }}
+          version=$(grep "^version = " pyproject.toml | sed -e 's/version = "\(.*\)"/\1/g')
+          if [ "${{ inputs.tag }}" != "${version}" ]; then
+            echo "The input tag does not match the version from pyproject.toml:" >&2
+            echo "${{ inputs.tag }}" >&2
+            echo "${version}" >&2
+            exit 1
+          else
+            echo "Releasing ${version}"
+          fi
 
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v4
+  tag-release:
+    name: Tag release
+    runs-on: ubuntu-latest
+    needs: validate-tag
+    # If you don't set an input tag, it's a dry run (no uploads).
+    if: ${{ inputs.tag }}
+    permissions:
+      # For git tag
+      contents: write
+    steps:
+      - uses: actions/checkout@v4
         with:
-          python-version: "3.10"
+          ref: ${{ inputs.branch }}
 
-      - name: Install hatch
+      - name: git tag
         run: |
-          pip install hatch
+          git config --global user.email "cergen@berkeley.edu"
+          git config --global user.name "popv release"
+          git tag -m "${{ inputs.tag }}" "${{ inputs.tag }}"
+          # If there is duplicate tag, this will fail. The publish to pypi action will have been a noop (due to skip
+          # existing), so we make a non-destructive exit here
+          git push --tags
+
+  publish-release:
+    name: Publish to GitHub
+    runs-on: ubuntu-latest
+    needs: tag-release
+    # If you don't set an input tag, it's a dry run (no uploads).
+    if: ${{ inputs.tag }}
+    permissions:
+      # For GitHub release publishing
+      contents: write
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.branch }}
+
+      - uses: softprops/action-gh-release@v2
+        with:
+          name: popv ${{ inputs.tag }}
+          tag_name: ${{ inputs.tag }}
+          generate_release_notes: false
+          body_path: .github/workflows/release_notes.md
+          prerelease: ${{ inputs.prerelease }}
+          target_commitish: ${{ inputs.branch }}
+
+  upload-release:
+    runs-on: ubuntu-latest
+    needs: tag-release
+    environment:
+      name: pypi
+      url: https://pypi.org/p/popv
+    permissions:
+      id-token: write # IMPORTANT: this permission is mandatory for trusted publishing
+    # If you don't set an input tag, it's a dry run (no uploads).
+    if: ${{ inputs.tag }}
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          filter: blob:none
+          fetch-depth: 0
+          ref: ${{ inputs.branch }}
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+
+      - run: pip install build
+
+      - run: python -m build
+
+      - uses: pypa/gh-action-pypi-publish@release/v1
+
+  build-image:
+    runs-on: ubuntu-latest
+    needs: tag-release
+
+    permissions:
+      contents: read
+      packages: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        dependencies: [""]
+    # If you don't set an input tag, it's a dry run (no uploads).
+    if: ${{ inputs.tag }}
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          filter: blob:none
+          fetch-depth: 0
+          ref: ${{ inputs.branch }}
 
-      - name: Build project for distribution
-        run: hatch build
+      - uses: docker/setup-buildx-action@v3
+
+      - uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Publish a Python distribution to PyPI
-        uses: pypa/gh-action-pypi-publish@release/v1
+      - uses: docker/build-push-action@v5
         with:
-          password: ${{ secrets.PYPI_TOKEN }}
+          context: .
+          file: ./Dockerfile
+          push: true
+          cache-from: type=registry,ref=ghcr.io/yoseflab/popv:buildcache
+          cache-to: type=inline,ref=ghcr.io/yoseflab/popv:buildcache
+          target: build
+          tags: ghcr.io/yoseflab/popv:py3.11-cu12-${{ inputs.tag }}-${{ matrix.dependencies }}
+          build-args: |
+            DEPENDENCIES=${{ matrix.dependencies }}

.github/workflows/release_notes.md

@@ -0,0 +1,9 @@
+See the [release notes](https://docs.popv.org/en/stable/changelog.html) for all changes.
+
+This release is available via PyPi:
+
+```bash
+pip install popv
+```
+
+Please report any issues on [GitHub](https://github.com/yoseflab/popv).

.github/workflows/test_linux_cuda.yml

@@ -30,7 +30,7 @@ jobs:
         shell: bash -e {0} # -e to fail on error
 
     container:
-      image: ghcr.io/yoseflab/popv:py3.10-cu12-base
+      image: ghcr.io/yoseflab/popv:py3.11-cu12-0.5.2.post1-
       options: --user root --gpus all --pull always
 
     name: integration

.pre-commit-config.yaml

@@ -18,7 +18,7 @@ repos:
         types: [yaml]
 
   - repo: https://github.com/executablebooks/mdformat
-    rev: 0.7.21
+    rev: 0.7.22
     hooks:
       - id: mdformat
         additional_dependencies:
@@ -31,7 +31,7 @@ repos:
           )$
 
   - repo: https://github.com/igorshubovych/markdownlint-cli
-    rev: v0.43.0
+    rev: v0.44.0
     hooks:
       - id: markdownlint-fix
         exclude: |
@@ -41,7 +41,7 @@ repos:
           )$
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.9.1
+    rev: v0.11.8
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]

.readthedocs.yaml

@@ -13,5 +13,5 @@ python:
         - docsbuild
 submodules:
   include:
-    - "docs/notebooks"
+    - "docs/tutorials/notebooks"
   recursive: true

CHANGELOG.md

@@ -1,15 +1,41 @@
-# Changelog
+<!-- markdownlint-disable MD024 -->
 
-All notable changes to this project will be documented in this file.
+# Release notes
 
-The format is based on [Keep a Changelog],
-and this project adheres to [Semantic Versioning].
+Starting from version 0.5.0, this format is based on [Keep a Changelog],
+and this project adheres
+to [Semantic Versioning]. Full commit history is available in the
+[commit logs](https://github.com/Yoseflab/popv/commits/).
 
-## [Unreleased]
+For the original manuscript version 0.4.0 was used. 0.5.0 is a
+major rewrite of the project to scale
+to several million cells on workstations with 128 GB of RAM
+and enable more GPU-accelerated
+functions.
 
-### Added
+## Version 0.5
 
-- Basic tool, preprocessing and plotting functions
+### 0.5.2
+
+#### Fixed
+
+- Changed Louvain overclustering for Celltypist to Leiden.
+- Fixed documentation references.
+- Fix release workflow.
+
+### 0.5.1
+
+#### Fixed
+
+- Fixed errors in Google Colab.
+
+### 0.5.0
+
+First major release. Production-ready code.
+
+### 0.4.0
+
+Version used for the manuscript.
 
 [keep a changelog]: https://keepachangelog.com/en/1.0.0/
 [semantic versioning]: https://semver.org/spec/v2.0.0.html

CODE_OF_CONDUCT.md

@@ -0,0 +1,18 @@
+FROM nvidia/cuda:12.4.0-runtime-ubuntu22.04
+FROM python:3.11 AS base
+
+RUN pip install --no-cache-dir uv
+
+RUN uv pip install --system --no-cache torch torchvision torchaudio "jax[cuda12]"
+RUN pip install --no-cache-dir --extra-index-url https://pypi.nvidia.com cuml-cu12 cudf-cu12 cugraph-cu12
+
+CMD ["/bin/bash"]
+
+FROM base AS build
+
+ENV POPV_PATH="/usr/local/lib/python3.11/site-packages/popv"
+
+COPY . ${POPV_PATH}
+
+ARG DEPENDENCIES=""
+RUN uv pip install --system "popv[${DEPENDENCIES}] @ ${POPV_PATH}"