Merge pull request #86 from octue/fix-test-matrix

Fix test matrix and update stack

Author: thclark

.devcontainer/devcontainer.json

@@ -12,39 +12,27 @@
     "vscode": {
       "settings": {
         "[python]": {
-          "editor.defaultFormatter": "ms-python.black-formatter",
           "editor.formatOnSave": true,
           "editor.codeActionsOnSave": {
             "source.organizeImports": "always",
             "source.fixAll": "always"
-          }
+          },
+          "editor.defaultFormatter": "charliermarsh.ruff"
         },
-        "black-formatter.args": ["--line-length", "120"],
-        "black-formatter.importStrategy": "fromEnvironment",
-        "isort.args": ["--profile", "black"],
-        "isort.importStrategy": "fromEnvironment",
         "austin.mode": "Wall time",
         "editor.defaultFormatter": "esbenp.prettier-vscode",
         "editor.formatOnSave": true,
-        "jupyter.widgetScriptSources": ["jsdelivr.com", "unpkg.com"],
-        // Line length to match black settings
-        // Disable specific messages:
-        //  - To find the details do: /usr/local/py-utils/bin/pylint --list-msgs
-        //  - Disable missing-module-docstring (C0114) because we don't document modules routinely, just their members
-        //  - Disable invalid-name (C0103) because pylint thinks that eg 'x', 'df', 'np' are invalid due to their lengths
-        "pylint.args": [
-          "--max-line-length=120",
-          "--disable=missing-module-docstring,invalid-name",
-          "--load-plugins=pylint_django"
-        ],
+        "python.locator": "native",
         "python.testing.pytestEnabled": true,
         "python.testing.unittestEnabled": false,
         "terminal.integrated.defaultProfile.linux": "zsh",
-        // Handle this: https://github.com/microsoft/vscode-python/issues/693
-        "python.testing.pytestArgs": ["--no-cov"]
+        "ruff.importStrategy": "fromEnvironment"
       },
-      // Add the IDs of extensions you want installed when the container is created.
+
+      // IN ALPHABETIC ORDER: Add the IDs of extensions you want installed when the container is created.
       "extensions": [
+        "4ops.terraform",
+        "charliermarsh.ruff",
         "erikphansen.vscode-toggle-column-selection",
         "esbenp.prettier-vscode",
         "GitHub.copilot",
@@ -53,38 +41,17 @@
         "irongeek.vscode-env",
         "ms-python.python",
         "ms-python.vscode-pylance",
-        "ms-python.pylint",
-        "ms-python.black-formatter",
-        "ms-python.flake8",
-        "ms-python.isort",
-        "ms-toolsai.jupyter",
-        "ms-toolsai.jupyter-renderers",
-        "ms-toolsai.jupyter-keymap",
         "ms-vsliveshare.vsliveshare",
+        "ninoseki.vscode-mogami",
         "p403n1x87.austin-vscode",
         "ritwickdey.liveserver",
-        "shamanu4.django-intellisense",
-        "thebarkman.vscode-djaneiro",
-        "trond-snekvik.simple-rst",
-        "4ops.terraform"
+        "tamasfe.even-better-toml"
       ]
     }
   },
 
   // Use 'forwardPorts' to make a list of ports inside the container available locally.
-  "forwardPorts": [
-    80,
-    443,
-    5000,
-    7045,
-    7046,
-    7047,
-    7048,
-    7049,
-    8000,
-    8001,
-    8080
-  ],
+  "forwardPorts": [80, 443, 5000, 7045, 7046, 7047, 7048, 7049, 8000, 8080],
 
   // Use 'postAttachCommand' to run commands after the container is created and attached (ie git is available).
   // This is preferential to postCreateCommand as invoking git (eg for private poetry installs or addition of
@@ -112,6 +79,13 @@
       "ppa": true,
       "version": "os-provided"
     },
+    "ghcr.io/devcontainers/features/git-lfs:1": {
+      "version": "latest"
+    },
+    "ghcr.io/devcontainers/features/github-cli:1": {
+      "installDirectlyFromGitHubRelease": true,
+      "version": "latest"
+    },
     "ghcr.io/devcontainers/features/node:1": {
       "nodeGypDependencies": false,
       "version": "latest"

.devcontainer/docker-compose.yml

@@ -18,6 +18,7 @@ services:
     environment:
       - DATABASE_URL=postgres://postgres_user:postgres_password@localhost/postgres_db
       - DJANGO_SETTINGS_MODULE=tests.server.settings
+      - DATABASE_ENGINE=postgres
 
     # Map the current directory containing code
     volumes:
@@ -50,9 +51,15 @@ services:
       - POSTGRES_DB=postgres_db
       - POSTGRES_USER=postgres_user
       - POSTGRES_PASSWORD=postgres_password
+    volumes:
+      - postgres_data:/var/lib/postgresql
     restart: unless-stopped
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U postgres"]
       interval: 10s
       timeout: 5s
       retries: 5
+
+volumes:
+  postgres_data:
+    driver: local

.devcontainer/postattach.sh

@@ -1,6 +1,8 @@
 #!/bin/zsh
 
-# Install dependencies
+
+# Install dependencies to a cache that persists across devcontainer builds
+poetry config cache-dir /workspace/.poetry_cache
 poetry install
 
 # Auto set up remote when pushing new branches
@@ -32,5 +34,7 @@ echo 'alias djmm="python manage.py makemigrations"' >> ~/.zshrc
 echo 'alias djm="python manage.py migrate"' >> ~/.zshrc
 echo 'alias djr="python manage.py runserver"' >> ~/.zshrc
 echo 'alias djreset="python manage.py reset_db -c"' >> ~/.zshrc
-echo 'alias djs="python manage.py shell_plus"' >> ~/.zshrc
+echo 'alias djsh="python manage.py shell_plus"' >> ~/.zshrc
+echo 'alias djsm="python manage.py showmigrations"' >> ~/.zshrc
+echo 'alias djt="pytest backend/test"' >> ~/.zshrc
 echo 'alias dju="python manage.py show_urls"' >> ~/.zshrc

.github/workflows/ci.yml

@@ -23,14 +23,70 @@ on:
         default: false
 
 jobs:
-  run-tests:
+  check-ahead-of-main:
+    if: github.ref != 'refs/heads/main'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Check branch is ahead of main
+        run: |
+          if ! git merge-base --is-ancestor origin/main ${{ github.event.pull_request.head.sha }};
+          then echo "::error::This branch is not up-to-date with the latest main branch commit.";
+          exit 1; fi
+
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install and configure poetry
+        uses: snok/install-poetry@v1
+        with:
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+
+      - name: Setup virtual environment cache
+        id: cached-poetry-dependencies
+        uses: actions/cache@v4
+        with:
+          path: .venv
+          key: venv-${{ runner.os }}-3.11-${{ hashFiles('**/poetry.lock') }}
+
+      - name: Install dependencies (if not cached)
+        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+        run: poetry install --only dev --no-interaction --no-root
+
+      - name: Install root project
+        run: poetry install --no-interaction
+
+      - name: Run precommit
+        run: SKIP=build-docs,check-branch-name poetry run pre-commit run --all-files
+
+  test:
     if: github.event.pull_request.draft == false
     strategy:
-      fail-fast: true
+      fail-fast: false
       matrix:
-        python: ["3.11", "3.12", "3.13"]
+        python-version: ["3.10", "3.11", "3.12"]
+        django-version: [">=4.2,<4.3", ">=5.0,<5.1"]
         os: [ubuntu-latest] # [ubuntu-latest, windows-latest, macos-latest] for full coverage but this gets expensive quickly
-    runs-on: ${{ matrix.os }}
+
+    permissions:
+      id-token: write
+      contents: read
+
+    env:
+      DJANGO_SETTINGS_MODULE: tests.server.settings
 
     services:
       postgres:
@@ -48,50 +104,71 @@ jobs:
           --health-timeout 5s
           --health-retries 5
 
+    runs-on: ${{ matrix.os }}
+
     steps:
-      - name: Checkout Repository
-        uses: actions/checkout@v3
+      - name: Check out repository
+        uses: actions/checkout@v4
 
-      - name: Setup Python ${{ matrix.python }}
-        uses: actions/setup-python@v3
+      - name: Set up python ${{ matrix.python-version }}
+        id: setup-python
+        uses: actions/setup-python@v5
         with:
-          python-version: ${{ matrix.python }}
+          python-version: ${{ matrix.python-version }}
 
-        # See the repo of this action for way more advanced caching strategies than used here
       - name: Install Poetry
         uses: snok/install-poetry@v1
+        with:
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+
+      - name: Setup virtual environment cache
+        id: cached-poetry-dependencies
+        uses: actions/cache@v4
+        with:
+          path: .venv
+          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
 
-        # For more advanced configuration see https://github.com/ymyzk/tox-gh-actions
-      - name: Install tox and plugins
+      - name: Install dependencies (if not cached)
+        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+        run: poetry install --no-interaction --no-root
+
+      - name: Install root project
+        run: poetry install --no-interaction
+
+      - name: Install django ${{ matrix.django-version }}
         run: |
-          python -m pip install --upgrade pip
-          python -m pip install tox==3.24.5 tox-gh-actions==2.9.1 tox-poetry==0.4.1
+          source .venv/bin/activate
+          pip install "django${{ matrix.django-version }}"
 
       - name: Setup tmate session [DEBUG]
         if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled == 'true'}}
         uses: mxschmitt/action-tmate@v3
 
-      # For more advanced configuration see https://github.com/ymyzk/tox-gh-actions
-      - name: Run tests using tox
-        run: tox
+      - name: Run tests
+        run: poetry run pytest --cov=django_gcp --cov-report=xml
 
       - name: Upload coverage to Codecov
         # This seems redundant inside the test matrix but actually isn't, since different
         # dependency combinations may cause different lines of code to be hit (e.g. backports)
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with:
           files: coverage.xml
           fail_ci_if_error: false
-          # Token is not required for public repos, but see:
+          # Token is not strictly required for public repos, but see:
           # https://community.codecov.com/t/upload-issues-unable-to-locate-build-via-github-actions-api/3954
           token: ${{ secrets.CODECOV_TOKEN }}
 
-  test-publish:
+  publish-test:
     runs-on: ubuntu-latest
-    needs: run-tests
+    needs:
+      - lint
+      - check-ahead-of-main
+
     permissions:
       id-token: write
       contents: read
+
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4