ci.yml

name: LAPIS-SILO

on:
  push:
    branches:
      - main
  pull_request:

env:
  DOCKER_DEPENDENCY_IMAGE_NAME: ghcr.io/genspectrum/lapis-silo-dependencies
  DOCKER_IMAGE_NAME: ghcr.io/genspectrum/lapis-silo

jobs:
  release-please:
    concurrency:
      group: lapis-silo-release-please-${{ github.ref }}
      cancel-in-progress: false
    runs-on: ubuntu-latest
    permissions:
      contents: write
      pull-requests: write
    steps:
      - uses: googleapis/release-please-action@v5
        if: github.event_name == 'push'
        id: release
        with:
          path: .
          config-file: release-please-config.json
          manifest-file: .release-please-manifest.json
          target-branch: main

      - name: Inspect Release Outputs
        if: github.event_name == 'push'
        run: |
          cat <<'EOF'
          ${{ toJson(steps.release.outputs) }}
          EOF

      - uses: actions/checkout@v7
        if: >-
          github.event_name == 'push'
          && steps.release.outputs.prs_created == 'true'
        with:
          fetch-depth: 0

      - name: Annotate changelog with serialization version change
        if: >-
          github.event_name == 'push'
          && steps.release.outputs.prs_created == 'true'
        run: |
          set -euo pipefail

          # PR_JSON is passed via the environment (see env: below) rather than
          # interpolated into the script, so single quotes or other shell
          # metacharacters in the payload cannot break parsing or inject commands.
          BRANCH=$(echo "$PR_JSON" | jq -r '.headBranchName // empty')
          VERSION=$(echo "$PR_JSON" | jq -r '.title // empty' | grep -oP '\d+\.\d+\.\d+' || true)

          if [ -z "$BRANCH" ] || [ -z "$VERSION" ]; then
            echo "Could not determine branch ($BRANCH) or version ($VERSION), skipping"
            exit 0
          fi

          git fetch origin "$BRANCH"
          git checkout -B "$BRANCH" "origin/$BRANCH"

          ./.github/scripts/annotate-changelog-serialization-version.sh \
            --version="$VERSION"
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          PR_JSON: ${{ steps.release.outputs.pr }}

      - uses: actions/checkout@v7
        if: >-
          github.event_name == 'push'
          && steps.release.outputs.release_created == 'true'
        with:
          fetch-depth: 0

      - name: Annotate GitHub release with serialization version change
        if: >-
          github.event_name == 'push'
          && steps.release.outputs.release_created == 'true'
        run: |
          ./.github/scripts/annotate-release-serialization-version.sh \
            --tag="${{ steps.release.outputs.tag_name }}"
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

  dependencyImage:
    name: Build dependency image (${{ matrix.arch }})
    strategy:
      matrix:
        arch: ["amd64", "arm64"]
    runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-22.04-arm' || 'ubuntu-latest' }}
    permissions:
      packages: write
    steps:
      - uses: actions/checkout@v7

      - name: Login to GitHub Container Registry
        uses: docker/login-action@v4
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v4

      - name: Determine HEAD_SHA
        run: |
          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
            echo "HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
          else
            echo "HEAD_SHA=${{ github.sha }}" >> $GITHUB_ENV
          fi

      - name: Generate dependency files hash
        run: |
          DIR_HASH=$(echo -n ${{ hashFiles('conanfile.py', 'buildScripts/install-dependencies', 'buildScripts/create-conanprofile', './Dockerfile_dependencies') }})
          echo "DIR_HASH=$DIR_HASH" >> $GITHUB_ENV

      - name: Check if dependency image exists
        run: |
          EXISTS=$(docker manifest inspect \
            ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:filehash-${{ env.DIR_HASH }}-${{ matrix.arch }} > /dev/null 2>&1 &&
            echo "true" || echo "false")
          echo "CACHE_HIT=$EXISTS" >> $GITHUB_ENV

      - name: Build and push dependency image if input files changed
        if: env.CACHE_HIT == 'false'
        uses: docker/build-push-action@v7
        with:
          context: .
          file: Dockerfile_dependencies
          push: true
          tags: ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:filehash-${{ env.DIR_HASH }}-${{ matrix.arch }}
          cache-from: type=gha,ref=dependency-image-cache-${{ matrix.arch }}-${{ hashFiles('conanfile.py', 'Dockerfile_dependencies') }}
          cache-to: type=gha,mode=min,ref=dependency-image-cache-${{ hashFiles('conanfile.py', 'Dockerfile_dependencies') }}

      - name: Tag dependency image with commit hash
        run: |
          docker buildx imagetools create \
            --tag ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-${{ matrix.arch }} \
            ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:filehash-${{ env.DIR_HASH }}-${{ matrix.arch }}

  formatting:
    name: Formatting Check
    needs: dependencyImage
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v7

      - name: Determine HEAD_SHA
        run: |
          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
            echo "HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
          else
            echo "HEAD_SHA=${{ github.sha }}" >> $GITHUB_ENV
          fi

      - name: Check formatting
        run: |
          docker run --rm \
            -v ${{ github.workspace }}:/workspace \
            -w /workspace \
            ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-amd64 \
            bash --login -c "make check-format"

  dockerImage:
    name: Build docker images
    needs: [dependencyImage, release-please]
    strategy:
      matrix:
        arch: ["amd64", "arm64"]
    runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-22.04-arm' || 'ubuntu-latest' }}
    permissions:
      packages: write
    steps:
      - uses: actions/checkout@v7

      - name: Login to GitHub Container Registry
        uses: docker/login-action@v4
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v4

      - name: Determine HEAD_SHA
        run: |
          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
            echo "HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
          else
            echo "HEAD_SHA=${{ github.sha }}" >> $GITHUB_ENV
          fi

      - name: Write release_version.txt
        run: |
          git fetch origin 'refs/tags/*:refs/tags/*'

          echo "Available tags:"
          git tag -l

          if git describe --tags --exact-match >/dev/null 2>&1; then
            echo "Using release version from git tag"
            VERSION=$(git describe --tags --exact-match | sed 's/^v//')
          else
            echo "Using release version from commit hash"
            VERSION="${{ env.HEAD_SHA }}"
          fi

          echo "Using release version: $VERSION"
          echo "$VERSION" > release_version.txt

      - name: Build unit test image
        uses: docker/build-push-action@v7
        with:
          context: .
          target: builder
          tags: builder
          load: true
          cache-from: type=gha,ref=image-cache-${{ github.ref_name }}-${{ matrix.arch }}
          cache-to: type=gha,mode=min,ref=image-cache-${{ github.ref_name }}-${{ matrix.arch }}
          build-args: |
            DEPENDENCY_IMAGE=${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-${{ matrix.arch }}

      - name: Run unit tests
        run: |
          docker run \
            --entrypoint "./silo_test" \
            builder
          docker run \
            --entrypoint "./silo_app_test" \
            builder

      - name: Build and push production image
        uses: docker/build-push-action@v7
        with:
          context: .
          push: true
          cache-from: type=gha,ref=image-cache-${{ github.ref_name }}-${{ matrix.arch }}
          cache-to: type=gha,mode=min,ref=image-cache-${{ github.ref_name }}-${{ matrix.arch }}
          tags: ${{ env.DOCKER_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-${{ matrix.arch }}
          build-args: |
            DEPENDENCY_IMAGE=${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-${{ matrix.arch }}

  multiPlatformImages:
    name: Create multi-platform images
    needs: dockerImage
    runs-on: ubuntu-latest
    permissions:
      packages: write
      contents: write
    steps:
      - uses: actions/checkout@v7

      - name: Login to GitHub Container Registry
        uses: docker/login-action@v4
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v4

      - name: Determine HEAD_SHA
        run: |
          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
            echo "HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
          else
            echo "HEAD_SHA=${{ github.sha }}" >> $GITHUB_ENV
          fi

      - name: Determine whether to use a release version
        run: |
          git fetch origin 'refs/tags/*:refs/tags/*'

          echo "Available tags:"
          git tag -l

          if git describe --tags --exact-match >/dev/null 2>&1; then
            echo "Using release version from git tag"
            VERSION=$(git describe --tags --exact-match | sed 's/^v//')
            echo "SILO_VERSION=${VERSION}" >> $GITHUB_ENV
          fi

      - name: Docker metadata
        id: dockerMetadataImage
        uses: docker/metadata-action@v6
        with:
          images: ${{ env.DOCKER_IMAGE_NAME }}
          tags: |
            type=ref,event=pr
            type=raw,value=${{ github.head_ref || github.ref_name }}
            type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
            type=raw,value=commit-${{ env.HEAD_SHA }}
            type=semver,value=${{ env.SILO_VERSION }},enable=${{ env.SILO_VERSION != '' }},pattern={{version}}
            type=semver,value=${{ env.SILO_VERSION }},enable=${{ env.SILO_VERSION != '' }},pattern={{major}}.{{minor}}

      - name: Tag images
        run: |
          TAGS=(${{ steps.dockerMetadataImage.outputs.tags }})
          for TAG in "${TAGS[@]}"; do
            docker buildx imagetools create --tag $TAG \
              "${{ env.DOCKER_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-arm64" \
              "${{ env.DOCKER_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-amd64"
          done

      - name: Extract binaries for release
        if: env.SILO_VERSION != ''
        run: |
          echo "Extracting binaries for version ${{ env.SILO_VERSION }}"
          mkdir -p artifacts
          cd artifacts

          for arch in amd64 arm64; do
            IMAGE="${{ env.DOCKER_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-$arch"
            PLATFORM="linux/$arch"
            echo "Extracting from $IMAGE ($PLATFORM)"

            binary="silo-linux-$arch"

            docker create --platform $PLATFORM --name extract-$arch $IMAGE
            docker cp extract-$arch:/app/silo "./$binary"
            docker rm extract-$arch

            chmod +x "./$binary"

            echo "Verifying $binary ..."
            [ -x "$binary" ] || { echo "  ✗ FAILED: File does not exist or is not executable"; exit 1; }
            file "$binary" | grep -q ELF || { echo "  ✗ FAILED: File is not an ELF binary"; exit 1; }
            echo "✓ Verified: $binary"

            sha256sum "$binary" > "$binary.sha256sum"
            # Create tar archive so that the file is executable when downloaded from GitHub Releases
            tar -czf "$binary.tar.gz" "$binary"
          done

          echo "Files in artifacts/ after extraction:"
          ls -alh .

      - name: Upload binaries to GitHub Release
        if: env.SILO_VERSION != ''
        uses: softprops/action-gh-release@v3
        with:
          files: |
            artifacts/*.tar.gz
            artifacts/*.sha256sum
          tag_name: v${{ env.SILO_VERSION }}
          fail_on_unmatched_files: true
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

  endToEndTests:
    name: Run End To End Tests
    needs: dockerImage
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v7

      - name: Use Node.js ${{ matrix.node-version }}
        uses: actions/setup-node@v7.0.0
        with:
          node-version: 18.x

      - name: Determine HEAD_SHA
        run: |
          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
            echo "HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
          else
            echo "HEAD_SHA=${{ github.sha }}" >> $GITHUB_ENV
          fi

      - uses: actions/cache@v6
        with:
          path: ~/.npm
          key: "${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}"

      - name: npm install
        run: cd endToEndTests && npm ci

      - name: Start Docker Container and preprocess data
        run: docker compose -f docker-compose-for-tests-preprocessing-from-ndjson.yml up
        env:
          SILO_IMAGE: ${{ env.DOCKER_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-amd64

      - name: Start Docker Container and run api
        run: docker compose -f docker-compose-for-tests-api.yml up -d --wait
        env:
          SILO_IMAGE: ${{ env.DOCKER_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-amd64

      - name: Run Tests
        run: cd endToEndTests && SILO_URL=localhost:8080 npm run test

  linterChanges:
    name: Build/Run linter on changed files
    needs: dependencyImage
    if: github.event_name == 'pull_request'
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v7
        with:
          fetch-depth: 0

      - name: Determine HEAD_SHA
        run: |
          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
            echo "HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
          else
            echo "HEAD_SHA=${{ github.sha }}" >> $GITHUB_ENV
          fi

      - name: Run linter script
        run: buildScripts/lint-changes ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-amd64

  linterAll:
    name: Build/Run linter on all files
    needs: dependencyImage
    if: >-
      github.event_name == 'push' ||
      (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'trigger-linter'))
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v7

      - name: Determine HEAD_SHA
        run: |
          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
            echo "HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
          else
            echo "HEAD_SHA=${{ github.sha }}" >> $GITHUB_ENV
          fi

      - name: Build all files with clang-tidy
        run: buildScripts/lint-all ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-amd64

  unitTests:
    needs: dependencyImage
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v7

      - name: Determine HEAD_SHA
        run: |
          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
            echo "HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
          else
            echo "HEAD_SHA=${{ github.sha }}" >> $GITHUB_ENV
          fi

      - name: Run tests in Docker
        run: |
          docker run --rm \
            -v ${{ github.workspace }}:/workspace \
            -w /workspace \
            ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-amd64 \
            bash --login -c "mv /src/conanprofile . && make test"

  pythonTests:
    needs: dependencyImage
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v7

      - name: Determine HEAD_SHA
        run: |
          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
            echo "HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
          else
            echo "HEAD_SHA=${{ github.sha }}" >> $GITHUB_ENV
          fi

      - name: Run Python tests in Docker
        run: |
          docker run --rm \
            -v ${{ github.workspace }}:/workspace \
            -w /workspace \
            ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-amd64 \
            bash --login -c "mv /src/conanprofile . && mv /src/build . && make python-tests"

  buildWheelsLinux:
    name: Build wheel (linux-${{ matrix.arch }})
    needs: [dependencyImage, release-please]
    strategy:
      matrix:
        arch: ["amd64", "arm64"]
    runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-22.04-arm' || 'ubuntu-latest' }}
    permissions:
      contents: write
      packages: read
    steps:
      - uses: actions/checkout@v7

      - name: Login to GitHub Container Registry
        uses: docker/login-action@v4
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Determine HEAD_SHA
        run: |
          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
            echo "HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
          else
            echo "HEAD_SHA=${{ github.sha }}" >> $GITHUB_ENV
          fi

      - name: Build wheels in Docker
        run: |
          docker run --rm \
            -v ${{ github.workspace }}:/workspace \
            -w /workspace \
            ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-${{ matrix.arch }} \
            bash --login -c "
              set -euo pipefail
              mv /src/conanprofile .

              # Build the C++ project once
              export CMAKE_BUILD_PARALLEL_LEVEL=4
              make build/Release/silo

              # Build wheels for all supported Python versions
              mkdir -p wheelhouse
              for pyversion in 3.11 3.12 3.13 3.14; do
                uv python install \$pyversion
                rm -rf dist
                uv build --wheel --python \$pyversion
                uv tool run auditwheel repair dist/*.whl -w wheelhouse/
              done
            "

      - name: Upload wheel as workflow artifact
        uses: actions/upload-artifact@v7
        with:
          name: wheel-linux-${{ matrix.arch }}
          path: wheelhouse/*.whl

      - name: Determine release version
        run: |
          git fetch origin 'refs/tags/*:refs/tags/*'
          if git describe --tags --exact-match >/dev/null 2>&1; then
            VERSION=$(git describe --tags --exact-match | sed 's/^v//')
            echo "SILO_VERSION=${VERSION}" >> $GITHUB_ENV
          fi

      - name: Upload wheel to GitHub release
        if: env.SILO_VERSION != ''
        uses: softprops/action-gh-release@v3
        with:
          files: wheelhouse/*.whl
          tag_name: v${{ env.SILO_VERSION }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

  buildWheelsMacos:
    name: Build wheel (macos-${{ matrix.arch }})
    needs: release-please
    strategy:
      matrix:
        include:
          - runner: macos-15
            arch: arm64
          - runner: macos-15-intel
            arch: x86_64
    runs-on: ${{ matrix.runner }}
    permissions:
      contents: write
    steps:
      - uses: actions/checkout@v7

      - uses: astral-sh/setup-uv@v7

      - name: Install Conan
        run: uv tool install conan

      - name: Cache Conan packages
        uses: actions/cache@v6
        with:
          path: ~/.conan2
          key: conan-macos-${{ matrix.arch }}-${{ hashFiles('conanfile.py', 'Dockerfile_dependencies') }}
          restore-keys: |
            conan-macos-${{ matrix.arch }}-

      - name: Build wheels for all supported Python versions
        run: make build-wheels

      - name: Upload wheel as workflow artifact
        uses: actions/upload-artifact@v7
        with:
          name: wheel-macos-${{ matrix.arch }}
          path: wheelhouse/*.whl

      - name: Determine release version
        run: |
          git fetch origin 'refs/tags/*:refs/tags/*'
          if git describe --tags --exact-match >/dev/null 2>&1; then
            VERSION=$(git describe --tags --exact-match | sed 's/^v//')
            echo "SILO_VERSION=${VERSION}" >> $GITHUB_ENV
          fi

      - name: Upload wheel to GitHub release
        if: env.SILO_VERSION != ''
        uses: softprops/action-gh-release@v3
        with:
          files: wheelhouse/*.whl
          tag_name: v${{ env.SILO_VERSION }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

  wasmBuild:
    name: Build WebAssembly
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v7

      # Must match the Emscripten version used to generate the Conan profile
      # (see buildScripts/create-wasm-conanprofile).
      - name: Set up Emscripten
        uses: mymindstorm/setup-emsdk@v16
        with:
          version: 6.0.2
          actions-cache-folder: emsdk-cache

      # setup-emsdk prepends $EMSDK/upstream/emscripten to PATH, which contains
      # a directory literally named `cmake` (Emscripten's own CMake Modules).
      # GNU Make bypasses the shell for simple recipe lines and execs commands
      # directly; on Linux this PATH search does not skip directory entries
      # and does not fall back to later PATH entries after the resulting
      # EACCES, so bare `cmake` invocations fail with "Permission denied"
      # instead of finding the real cmake binary. Re-prioritize the system
      # cmake's directory so it is found first again.
      - name: Ensure system cmake takes priority over Emscripten's cmake/ directory
        run: echo "$(dirname "$(command -v cmake)")" >> "$GITHUB_PATH"

      - uses: astral-sh/setup-uv@v7

      # The WASM dependencies (Arrow, Boost, ...) are compiled from source for
      # Emscripten, which is slow. Cache the Conan package cache to avoid
      # rebuilding them on every run.
      - name: Cache Conan packages
        uses: actions/cache@v6
        with:
          path: ~/.conan2
          key: conan-wasm-${{ hashFiles('conanfile.py', 'buildScripts/install-wasm-dependencies', 'buildScripts/create-wasm-conanprofile') }}
          restore-keys: |
            conan-wasm-

      - name: Use Node.js
        uses: actions/setup-node@v7.0.0
        with:
          node-version: latest

      - name: Build and test WASM target
        run: make wasm-test

      - name: Upload WASM artifacts
        uses: actions/upload-artifact@v7
        with:
          name: silo-wasm
          path: dist/wasm/*