ci.yml

# JULEA - Flexible storage framework
# Copyright (C) 2019-2026 Michael Kuhn
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

name: CI
on: [push, pull_request]
defaults:
  run:
    shell: bash
jobs:
  style:
    name: Style
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    steps:
      - name: Checkout
        uses: actions/checkout@v6
        with:
          persist-credentials: false
          show-progress: false
      - name: Check format
        run: |
          ./scripts/format.sh
      - name: Check shell scripts
        run: |
          ./scripts/check.sh
  dependencies:
    name: Dependencies
    runs-on: ubuntu-24.04
    container: ${{ matrix.os.registry }}/${{ matrix.os.image }}
    timeout-minutes: 180
    strategy:
      fail-fast: false
      matrix:
        os:
          - image: ubuntu:24.04-base
            registry: ghcr.io/parcio/julea
            compiler: gcc
            compiler_spack: gcc
            compiler_version: 13.3.0
          - image: ubuntu:24.04-base
            registry: ghcr.io/parcio/julea
            compiler: clang
            compiler_spack: llvm
            compiler_version: 18.1.3
    env:
      JULEA_SPACK_COMPILER: ${{ matrix.os.compiler_spack }}@${{ matrix.os.compiler_version }}
      JULEA_SPACK_DIR: /julea-dependencies
    steps:
      - name: Checkout
        uses: actions/checkout@v6
        with:
          persist-credentials: false
          show-progress: false
      - name: Cache dependencies
        id: cache
        uses: actions/cache@v5
        with:
          path: ${{ env.JULEA_SPACK_DIR }}
          key: ${{ matrix.os.image }}-${{ matrix.os.compiler }}-${{ matrix.os.compiler_version }}-${{ hashFiles('scripts/spack', 'scripts/install-dependencies.sh') }}
      - name: Install dependencies
        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
        run: |
          ./scripts/install-dependencies.sh
  build:
    name: Build
    needs: dependencies
    runs-on: ubuntu-24.04
    container: ${{ matrix.os.registry }}/${{ matrix.os.image }}
    timeout-minutes: 60
    strategy:
      fail-fast: false
      matrix:
        os:
          - image: ubuntu:24.04-base
            registry: ghcr.io/parcio/julea
            compiler: gcc
            compiler_spack: gcc
            compiler_version: 13.3.0
          - image: ubuntu:24.04-base
            registry: ghcr.io/parcio/julea
            compiler: clang
            compiler_spack: llvm
            compiler_version: 18.1.3
        dependencies: [system, spack]
    env:
      JULEA_SPACK_DIR: /julea-dependencies
    steps:
      - name: Checkout
        uses: actions/checkout@v6
        with:
          persist-credentials: false
          show-progress: false
      - name: Install dependencies
        if: ${{ matrix.dependencies == 'system' }}
        run: |
          apt update
          apt --yes --no-install-recommends install meson ninja-build pkgconf libglib2.0-dev libbson-dev libfabric-dev libgdbm-dev liblmdb-dev libsqlite3-dev libleveldb-dev libmongoc-dev libmariadb-dev librocksdb-dev libfuse3-dev libopen-trace-format-dev librados-dev
      - name: Cache dependencies
        id: cache
        if: ${{ matrix.dependencies == 'spack' }}
        uses: actions/cache/restore@v5
        with:
          path: ${{ env.JULEA_SPACK_DIR }}
          key: ${{ matrix.os.image }}-${{ matrix.os.compiler }}-${{ matrix.os.compiler_version }}-${{ hashFiles('scripts/spack', 'scripts/install-dependencies.sh') }}
      - name: Check dependencies
        if: ${{ matrix.dependencies == 'spack' && steps.cache.outputs.cache-hit != 'true' }}
        run: |
          exit 1
      - name: Configure, build and install
        env:
          CC: ${{ matrix.os.compiler }}
        run: |
          ./scripts/ci/build.sh release "${{ matrix.dependencies }}"
  tests:
    name: Tests
    needs: dependencies
    runs-on: ubuntu-24.04
    container: ${{ matrix.os.registry }}/${{ matrix.os.image }}
    timeout-minutes: 60
    strategy:
      fail-fast: false
      matrix:
        os:
          - image: ubuntu:24.04-base
            registry: ghcr.io/parcio/julea
            compiler: gcc
            compiler_spack: gcc
            compiler_version: 13.3.0
          - image: ubuntu:24.04-base
            registry: ghcr.io/parcio/julea
            compiler: clang
            compiler_spack: llvm
            compiler_version: 18.1.3
        dependencies: [system, spack]
        julea:
          # Default
          - object: posix
            kv: lmdb
            db: sqlite
          # Object backends
          - object: gio
            kv: lmdb
            db: sqlite
          # KV backends
          - object: posix
            kv: gdbm
            db: sqlite
          - object: posix
            kv: mongodb
            db: sqlite
          - object: posix
            kv: leveldb
            db: sqlite
          - object: posix
            kv: rocksdb
            db: sqlite
          - object: posix
            kv: sqlite
            db: sqlite
          # DB backends
          - object: posix
            kv: lmdb
            db: mysql
            db-server: mysql
          - object: posix
            kv: lmdb
            db: mysql
            db-server: mariadb
        exclude:
          # FIXME Ubuntu 24.04's RocksDB triggers asan
          - os:
              image: ubuntu:24.04-base
            dependencies: system
            julea:
              kv: rocksdb
    services:
      mariadb:
        image: ${{ matrix.julea.db == 'mysql' && matrix.julea.db-server == 'mariadb' && 'docker.io/library/mariadb' || '' }}
        env:
          MARIADB_RANDOM_ROOT_PASSWORD: yes
          MARIADB_DATABASE: juleadb
          MARIADB_USER: julea
          MARIADB_PASSWORD: aeluj
      mongodb:
        image: ${{ matrix.julea.kv == 'mongodb' && 'docker.io/library/mongo' || '' }}
      mysql:
        image: ${{ matrix.julea.db == 'mysql' && matrix.julea.db-server == 'mysql' && 'docker.io/library/mysql' || '' }}
        env:
          MYSQL_RANDOM_ROOT_PASSWORD: yes
          MYSQL_DATABASE: juleadb
          MYSQL_USER: julea
          MYSQL_PASSWORD: aeluj
    env:
      JULEA_SPACK_DIR: /julea-dependencies
    steps:
      - name: Checkout
        uses: actions/checkout@v6
        with:
          persist-credentials: false
          show-progress: false
      - name: Install dependencies
        if: ${{ matrix.dependencies == 'system' }}
        run: |
          apt update
          apt --yes --no-install-recommends install meson ninja-build pkgconf libglib2.0-dev libbson-dev libfabric-dev libgdbm-dev liblmdb-dev libsqlite3-dev libleveldb-dev libmongoc-dev libmariadb-dev librocksdb-dev libfuse3-dev libopen-trace-format-dev librados-dev
      - name: Cache dependencies
        id: cache
        if: ${{ matrix.dependencies == 'spack' }}
        uses: actions/cache/restore@v5
        with:
          path: ${{ env.JULEA_SPACK_DIR }}
          key: ${{ matrix.os.image }}-${{ matrix.os.compiler }}-${{ matrix.os.compiler_version }}-${{ hashFiles('scripts/spack', 'scripts/install-dependencies.sh') }}
      - name: Check dependencies
        if: ${{ matrix.dependencies == 'spack' && steps.cache.outputs.cache-hit != 'true' }}
        run: |
          exit 1
      - name: Configure and build
        env:
          CC: ${{ matrix.os.compiler }}
        run: |
          ./scripts/ci/build.sh debug "${{ matrix.dependencies }}"
      - name: Create configuration
        run: |
          . scripts/environment.sh
          JULEA_KV_PATH="/tmp/julea/kv/${{ matrix.julea.kv }}"
          if test "${{ matrix.julea.kv }}" = 'mongodb'; then JULEA_KV_PATH='mongodb:juleadb'; fi
          JULEA_DB_PATH="/tmp/julea/db/${{ matrix.julea.db }}"
          if test "${{ matrix.julea.db }}" = 'mysql'; then JULEA_DB_PATH='${{ matrix.julea.db-server }}:juleadb:julea:aeluj'; fi
          julea-config --user --object-servers="$(hostname)" --kv-servers="$(hostname)" --db-servers="$(hostname)" --object-backend="${{ matrix.julea.object }}" --object-path="/tmp/julea/object/${{ matrix.julea.object }}" --kv-backend="${{ matrix.julea.kv }}" --kv-path="${JULEA_KV_PATH}" --db-backend="${{ matrix.julea.db }}" --db-path="${JULEA_DB_PATH}"
      - name: Tests
        run: |
          . scripts/environment.sh
          ./scripts/setup.sh start
          ./scripts/test.sh
          sleep 10
          ./scripts/test.sh
          ./scripts/setup.sh stop
      - name: HDF5 Tests
        env:
          LSAN_OPTIONS: exitcode=0
        run: |
          . scripts/environment.sh
          ./scripts/setup.sh start
          export HDF5_VOL_CONNECTOR=julea-kv
          ./scripts/test.sh -r /hdf5
          sleep 10
          ./scripts/test.sh -r /hdf5
          sleep 10
          export HDF5_VOL_CONNECTOR=julea-db
          ./scripts/test.sh -r /hdf5
          sleep 10
          ./scripts/test.sh -r /hdf5
          ./scripts/setup.sh stop
      - name: Benchmarks
        if: ${{ matrix.julea.object == 'posix' && matrix.julea.kv == 'lmdb' && matrix.julea.db == 'sqlite' }}
        run: |
          . scripts/environment.sh
          ./scripts/setup.sh start
          ./scripts/benchmark.sh --duration=0
          sleep 10
          ./scripts/benchmark.sh --duration=0
          ./scripts/setup.sh stop
  doxygen:
    name: Doxygen
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    steps:
      - name: Checkout
        uses: actions/checkout@v6
        with:
          persist-credentials: false
          show-progress: false
      - name: Install dependencies
        run: |
          sudo apt update || true
          sudo apt --yes --no-install-recommends install doxygen
      - name: Generate documentation
        run: |
          doxygen
      - name: Upload doxygen
        uses: actions/upload-artifact@v7
        with:
          name: doxygen
          path: html
  coverage:
    # FIXME Reduce redundancies
    # FIXME Increase coverage (currently, only one configuration runs)
    name: Coverage
    needs: dependencies
    runs-on: ubuntu-24.04
    container: ghcr.io/parcio/julea/ubuntu:24.04-base
    timeout-minutes: 60
    env:
      JULEA_SPACK_DIR: /julea-dependencies
    steps:
      - name: Checkout
        uses: actions/checkout@v6
        with:
          persist-credentials: false
          show-progress: false
      - name: Cache dependencies
        id: cache
        uses: actions/cache/restore@v5
        with:
          path: ${{ env.JULEA_SPACK_DIR }}
          key: ubuntu:24.04-base-gcc-13.3.0-${{ hashFiles('scripts/spack', 'scripts/install-dependencies.sh') }}
      - name: Check dependencies
        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
        run: |
          exit 1
      - name: Configure and build
        env:
          CC: gcc
        run: |
          ./scripts/ci/build.sh coverage spack
      - name: Create configuration
        run: |
          . scripts/environment.sh
          julea-config --user --object-servers="$(hostname)" --kv-servers="$(hostname)" --db-servers="$(hostname)" --object-backend=posix --object-path=/tmp/julea/object/posix --kv-backend=lmdb --kv-path=/tmp/julea/kv/lmdb --db-backend=sqlite --db-path=/tmp/julea/db/sqlite
      - name: Tests
        env:
          HDF5_VOL_CONNECTOR: julea-db
          LSAN_OPTIONS: exitcode=0
        run: |
          . scripts/environment.sh
          ./scripts/setup.sh start
          ./scripts/test.sh
          ./scripts/setup.sh stop
      - name: Generate coverage
        run: |
          . scripts/environment.sh
          ninja -C bld coverage
      - name: Upload coverage
        uses: actions/upload-artifact@v7
        with:
          name: coverage
          path: bld/meson-logs/coveragereport
  publish:
    name: Publish
    needs: [doxygen, coverage]
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    steps:
      - name: Configure Pages
        uses: actions/configure-pages@v6
      - name: Download doxygen
        uses: actions/download-artifact@v8
        with:
          name: doxygen
          path: doxygen
      - name: Download coverage
        uses: actions/download-artifact@v8
        with:
          name: coverage
          path: coverage
      - name: Prepare
        run: |
          mv coverage doxygen/coverage
      - name: Upload artifact
        uses: actions/upload-pages-artifact@v5
        with:
          path: doxygen
  deploy:
    name: Deploy
    needs: publish
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    if: ${{ github.ref == 'refs/heads/master' }}
    permissions:
      id-token: write
      pages: write
    environment:
      name: github-pages
      url: ${{ steps.deployment.outputs.page_url }}
    steps:
      - name: Deploy Pages
        id: deployment
        uses: actions/deploy-pages@v5