Nextest-2024

.config/nextest.toml

@@ -0,0 +1,8 @@
+[test-groups]
+integration-tests = { max-threads = 1 }
+
+
+[[profile.default.overrides]]
+filter = 'package(geoengine-services)'
+test-group = 'integration-tests'
+slow-timeout = { period = "360s" }

.github/actions/test/action.yml

@@ -0,0 +1,6 @@
+name: "Run Tests & Generate Coverage"
+description: "Run Geo Engine tests with coverage"
+runs:
+  using: "docker"
+  image: "docker://quay.io/geoengine/devcontainer:latest"
+  entrypoint: "./.github/actions/test/test.sh"

.github/actions/test/test.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+function print_headline() {
+    local BOLD_WHITE_ON_CYAN="\e[1;46;37m"
+    local BOLD_CYAN="\e[1;49;36m"
+    local RESET_COLOR="\e[0m"
+    printf "${BOLD_WHITE_ON_CYAN} ▶ ${BOLD_CYAN} $1 ${RESET_COLOR}\n" >&2
+}
+
+print_headline "Install cargo-llvm-cov and nextest"
+cargo install --locked cargo-llvm-cov
+cargo install --locked cargo-nextest
+
+print_headline "Run Tests & Generate Code Coverage"
+service postgresql start
+cargo llvm-cov nextest-archive \
+    --archive-file tests.tar.zst \
+    --cargo-profile ci \
+    --locked \
+    --all-features
+cargo clean
+cargo llvm-cov nextest \
+    --archive-file tests.tar.zst
+    --all-features \
+    --lcov \
+    --output-path lcov.info
+
+print_headline "Run Doctests"
+# cf. https://github.com/taiki-e/cargo-llvm-cov/issues/2
+cargo test --doc --all-features --locked

.github/workflows/ci.yml

@@ -11,29 +11,11 @@ env:
 
 jobs:
   test:
-    runs-on: ${{matrix.os}}
-
-    services:
-      postgres:
-        image: postgis/postgis
-        env:
-          POSTGRES_USER: geoengine
-          POSTGRES_PASSWORD: geoengine
-          POSTGRES_DB: geoengine
-        ports:
-          - 5432:5432
-        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
-
-    strategy:
-      matrix:
-        os:
-          - ubuntu-22.04
-        features: [""]
-        build: [""]
+    runs-on: ubuntu-24.04
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Free Disk Space (Ubuntu)
         uses: jlumbroso/free-disk-space@main
         with:
@@ -44,22 +26,41 @@ jobs:
           large-packages: true
           docker-images: true
           swap-storage: true
-      - name: Install lld & GDAL & Protobuf
-        run: |
-          sudo apt-get update
-          sudo apt-get install lld libgdal-dev gdal-bin build-essential clang curl protobuf-compiler
-          sudo apt-get clean
-          export C_INCLUDE_PATH=/usr/include/gdal:$C_INCLUDE_PATH
-          export CPLUS_INCLUDE_PATH=/usr/include/gdal:$CPLUS_INCLUDE_PATH
-          sudo ldconfig
-      - name: Install Rustup
+      # TODO: Get Cache back
+      # - name: setup rust build cache
+      #   uses: Swatinem/rust-cache@v2
+      #   with:
+      #     # An explicit cache key that is used instead of the automatic `job`-based
+      #     # cache key and is thus stable across jobs.
+      #     # Default: empty
+      #     shared-key: ""
+
+      #     # An additional cache key that is added alongside the automatic `job`-based
+      #     # cache key and can be used to further differentiate jobs.
+      #     # Default: empty
+      #     key: ci_test_
+      - name: Testsuite
+        uses: ./.github/actions/test
+      - name: Adjust lcov.info
+        # The lcov file is generated in the container and contains absolute paths, so we need to adjust them.
         run: |
-          curl --proto '=https' --tlsv1.2 --retry 10 --retry-connrefused -fsSL "https://sh.rustup.rs" | sh -s -- --profile minimal --default-toolchain none -y
-          echo "${CARGO_HOME:-$HOME/.cargo}/bin" >> $GITHUB_PATH
-      - name: Install SQLFluff
-        run: pip install sqlfluff==3.3.0
+          sed -i 's|/github/workspace/||g' lcov.info
+      - name: Upload coverage to Coveralls
+        uses: coverallsapp/github-action@v2
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          file: lcov.info
+
+  lint:
+    runs-on: ubuntu-24.04
+    container: quay.io/geoengine/devcontainer:latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
       - name: Init rustup toolchain
-        run: rustup show # enough to initialize the toolchain
+        # somehow rustup show will take care to initialize the version based on the toolchain file
+        run: rustup show
       - name: setup rust build cache
         uses: Swatinem/rust-cache@v2
         with:
@@ -71,64 +72,30 @@ jobs:
           # An additional cache key that is added alongside the automatic `job`-based
           # cache key and can be used to further differentiate jobs.
           # Default: empty
-          key: ci_test_${{ matrix.os }}_${{ matrix.features }}_${{ matrix.build }}
-
-          # A whitespace separated list of env-var *prefixes* who's value contributes
-          # to the environment cache key.
-          # The env-vars are matched by *prefix*, so the default `RUST` var will
-          # match all of `RUSTC`, `RUSTUP_*`, `RUSTFLAGS`, `RUSTDOC_*`, etc.
-          # Default: "CARGO CC CFLAGS CXX CMAKE RUST"
-          #env-vars: ""
-
-          # The cargo workspaces and target directory configuration.
-          # These entries are separated by newlines and have the form
-          # `$workspace -> $target`. The `$target` part is treated as a directory
-          # relative to the `$workspace` and defaults to "target" if not explicitly given.
-          # Default: ". -> target"
-          #workspaces: ""
-
-          # Determines if the cache should be saved even when the workflow has failed.
-          # Default: "false"
-          #cache-on-failure: ""
-      - name: Check with Rustfmt
+          key: ci_test_
+      - name: Rustfmt
         run: cargo fmt --all -- --check
-      - name: Check with Clippy
-        run: cargo clippy --all-targets --locked ${{ matrix.features }} ${{ matrix.build }} -- -D warnings
-      - name: Check with SQLFluff
-        run: sqlfluff lint
-      - name: Verify for expression dependencies workspace
+      - name: Clippy
+        run: cargo clippy --all-targets --locked -- -D warnings
+      - name: SQLFluff
+        run: pipx run sqlfluff==3.3.0 lint
+      - name: Verifying expression dependencies workspace
         run: |
           rustup toolchain install nightly
           chmod +x ./.scripts/check-expression-deps.rs
           ./.scripts/check-expression-deps.rs
-      - name: Run tests
-        run: cargo test --locked ${{ matrix.features }} ${{ matrix.build }} --verbose
 
   build:
-    runs-on: ${{matrix.os}}
+    runs-on: ubuntu-24.04
+    container: quay.io/geoengine/devcontainer:latest
 
     strategy:
       matrix:
-        os:
-          - ubuntu-22.04
-        features: [""]
         build: ["", "--release"]
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
-      - name: APT update
-        run: sudo apt-get update
-      - name: Install lld & GDAL & Protobuf
-        run: |
-          sudo apt-get install lld libgdal-dev gdal-bin build-essential curl protobuf-compiler
-          export C_INCLUDE_PATH=/usr/include/gdal:$C_INCLUDE_PATH
-          export CPLUS_INCLUDE_PATH=/usr/include/gdal:$CPLUS_INCLUDE_PATH
-          sudo ldconfig
-      - name: Install Rustup
-        run: |
-          curl --proto '=https' --tlsv1.2 --retry 10 --retry-connrefused -fsSL "https://sh.rustup.rs" | sh -s -- --profile minimal --default-toolchain none -y
-          echo "${CARGO_HOME:-$HOME/.cargo}/bin" >> $GITHUB_PATH
+        uses: actions/checkout@v4
       - name: Init rustup toolchain
         # somehow rustup show will take care to initialize the version based on the toolchain file
         run: rustup show
@@ -143,24 +110,6 @@ jobs:
           # An additional cache key that is added alongside the automatic `job`-based
           # cache key and can be used to further differentiate jobs.
           # Default: empty
-          key: ci_test_${{ matrix.os }}_${{ matrix.features }}_${{ matrix.build }}
-
-          # A whitespace separated list of env-var *prefixes* who's value contributes
-          # to the environment cache key.
-          # The env-vars are matched by *prefix*, so the default `RUST` var will
-          # match all of `RUSTC`, `RUSTUP_*`, `RUSTFLAGS`, `RUSTDOC_*`, etc.
-          # Default: "CARGO CC CFLAGS CXX CMAKE RUST"
-          #env-vars: ""
-
-          # The cargo workspaces and target directory configuration.
-          # These entries are separated by newlines and have the form
-          # `$workspace -> $target`. The `$target` part is treated as a directory
-          # relative to the `$workspace` and defaults to "target" if not explicitly given.
-          # Default: ". -> target"
-          #workspaces: ""
-
-          # Determines if the cache should be saved even when the workflow has failed.
-          # Default: "false"
-          #cache-on-failure: ""
+          key: ci_test_${{ matrix.build }}
       - name: Build
-        run: cargo build --locked ${{ matrix.features }} ${{ matrix.build }} --verbose
+        run: cargo build --locked ${{ matrix.build }} --verbose

.github/workflows/clear-cache.yml

@@ -9,7 +9,7 @@ env:
 
 jobs:
   clear-cache:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
 
     steps:
       - name: Clear all caches
@@ -21,7 +21,7 @@ jobs:
             const [owner, repo] = GITHUB_REPOSITORY.split('/')
 
             console.log(`Clearing all caches for ${owner}/${repo}`)
-            
+
             let caches = [];
 
             do {

.github/workflows/containers.yml

@@ -25,7 +25,7 @@ jobs:
   create-container:
     if: github.event_name != 'schedule' || github.repository_owner == 'geo-engine'
 
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
 
     env:
       TAG_NAME: nightly
@@ -54,13 +54,13 @@ jobs:
           echo "CONTAINER_REPOSITORY_BRANCH=${{ github.event.inputs.container_repository_branch }}" >> $GITHUB_ENV
 
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: ${{ github.ref_name }}
           path: geoengine
 
       - name: Checkout container files
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: geo-engine/geoengine-container
           ref: ${{ env.CONTAINER_REPOSITORY_BRANCH }}
@@ -98,7 +98,7 @@ jobs:
     name: Post to a Slack channel in case of failure
     needs: create-container
     if: always()
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Post to a Slack channel
         if: ${{ needs.create-container.result == 'failure' }}