Merge pull request #79 from ScilifelabDataCentre/gh-actions-release-strat

One GH action for making a new release

Author: RMCrean

.github/workflows/publish-to-pypi.yaml

@@ -1,7 +1,7 @@
 name: Publish divbase-lib and divbase-cli packages to Test PyPI
 on:
   workflow_dispatch:
-
+  
 jobs:
   build-divbase-lib-and-cli:
     uses: ./.github/workflows/build-pypi-packages.yaml

.github/workflows/publish-to-test-pypi.yaml

@@ -0,0 +1,95 @@
+name: Build and push DEV images to ghcr and run pytest on the images
+
+on: 
+  # NOTE: for a release build of the images, see release.yaml. 
+  # The workflow_dispatch can be used for dev images
+  # images are tagged with "latest-dev" and the commit hash
+  # This workflow builds and publishes the images before running the tests
+  # so you don't have to wait to put them on the dev cluster
+  # For the release builds, we run tests before building the images.
+  workflow_dispatch:
+
+jobs:
+  push-to-container-registry:
+    if: github.repository == 'ScilifelabDataCentre/divbase'  
+    name: Build and push DEV images to ghcr
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    strategy:
+      matrix:
+        include:
+          - dockerfile: fastapi.dockerfile
+            image_name: divbase-fastapi
+          - dockerfile: worker.dockerfile
+            image_name: divbase-worker
+            
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v6
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v4
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      
+      - name: Build and push Docker image 
+        uses: docker/build-push-action@v6
+        with:
+            push: true
+            context: .
+            file: ./docker/${{ matrix.dockerfile }}
+            tags: |
+                ghcr.io/scilifelabdatacentre/${{ matrix.image_name }}:${{ github.sha }}
+                ghcr.io/scilifelabdatacentre/${{ matrix.image_name }}:latest-dev
+
+  run-pytest-on-newly-built-images:
+    name: Run pytest on the newly built DEV images
+    needs: push-to-container-registry
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v6
+      
+      - name: Install bcftools # some test set up data calling bcftools directly
+        run: | 
+          sudo apt-get update  
+          sudo apt-get install autoconf automake make gcc perl zlib1g-dev libbz2-dev liblzma-dev libcurl4-gnutls-dev libssl-dev libperl-dev libgsl0-dev
+          git clone --recurse-submodules https://github.com/samtools/htslib.git
+          git clone https://github.com/samtools/bcftools.git
+          cd bcftools
+          make
+          sudo make install 
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          version: "0.9.15"
+          enable-cache: true
+
+      - name: Install the project
+        run: | 
+          uv sync --locked --dev
+          uv run playwright install chromium
+
+      - name: Pull new images built in the previous workflow run
+        run: |
+          docker pull ghcr.io/scilifelabdatacentre/divbase-fastapi:${{ github.sha }}
+          docker pull ghcr.io/scilifelabdatacentre/divbase-worker:${{ github.sha }}
+
+      - name: Allow container to write to test fixtures # actions runner uid=1001, but container runs as 1000 
+        # So container does not have permissions to write to the mounted test fixtures, required by test_bcftools_pipe_cli_integration_with_eager_mode 
+        # volume mount made in docker/divbase_compose.tests.yaml 
+        run: chmod -R a+w tests/fixtures
+
+      - name: Run pytest including slow tests
+        env: 
+          GITHUB_ACTIONS_RUNNER: "true"
+          DIVBASE_FASTAPI_IMAGE: ghcr.io/scilifelabdatacentre/divbase-fastapi:${{ github.sha }}
+          DIVBASE_WORKER_IMAGE: ghcr.io/scilifelabdatacentre/divbase-worker:${{ github.sha }}
+        # use of env vars is to make click/typer think it is running in a terminal 
+        run: FORCE_TERMINAL=1 COLUMNS=1000 uv run pytest --run-slow

.github/workflows/publish_dev_images.yaml

@@ -1,7 +1,6 @@
 name: Build and deploy the divbase documentation website
 on:
-  release:
-    types: [published]
+  # NOTE: A new release will auto publish a new version of the docs, see release.yaml.
   workflow_dispatch:
 
 permissions:
@@ -23,11 +22,5 @@ jobs:
       - name: Install the project
         run: uv sync --locked --dev
 
-      - name: Cache mkdocs deps
-        uses: actions/cache@v5
-        with: 
-          key: ${{ github.ref }}
-          path: .cache
-
       - name: Deploy documentation
         run: uv run mkdocs gh-deploy --force