ci.yml

name: LAPIS-SILO

on:
  pull_request:
  workflow_run:
    workflows: [Release SILO]
    types: [completed]
    branches: [main]

env:
  DOCKER_DEPENDENCY_IMAGE_NAME: ghcr.io/genspectrum/lapis-silo-dependencies
  DOCKER_IMAGE_NAME: ghcr.io/genspectrum/lapis-silo

jobs:
  dockerImage:
    name: Build docker images
    strategy:
      matrix:
        arch: ["amd64", "arm64"]
    runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-22.04-arm' || 'ubuntu-latest' }}
    permissions:
      packages: write
    steps:
      - uses: actions/checkout@v6

      - name: Login to GitHub Container Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Determine HEAD_SHA
        run: |
          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
            echo "HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
          else
            echo "HEAD_SHA=${{ github.sha }}" >> $GITHUB_ENV
          fi

      - name: Generate dependency files hash
        run: |
          DIR_HASH=$(echo -n ${{ hashFiles('conanfile.py', 'conanprofile.docker', './Dockerfile_dependencies') }})
          echo "DIR_HASH=$DIR_HASH" >> $GITHUB_ENV

      - name: Check if dependency image exists
        run: |
          EXISTS=$(docker manifest inspect \
            ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:filehash-${{ env.DIR_HASH }}-${{ matrix.arch }} > /dev/null 2>&1 && 
            echo "true" || echo "false")
          echo "CACHE_HIT=$EXISTS" >> $GITHUB_ENV

      - name: Build and push dependency image if input files changed
        if: env.CACHE_HIT == 'false'
        uses: docker/build-push-action@v6
        with:
          context: .
          file: Dockerfile_dependencies
          push: true
          tags: ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:filehash-${{ env.DIR_HASH }}-${{ matrix.arch }}
          cache-from: type=gha,ref=dependency-image-cache-${{ matrix.arch }}-${{ hashFiles('conanfile.py', 'Dockerfile') }}
          cache-to: type=gha,mode=min,ref=dependency-image-cache-${{ hashFiles('conanfile.py', 'Dockerfile') }}

      - name: Tag dependency image with commit hash
        run: |
          docker buildx imagetools create \
            --tag ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-${{ matrix.arch }} \
            ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:filehash-${{ env.DIR_HASH }}-${{ matrix.arch }}

      - name: Write release_version.txt
        run: |
          git fetch origin 'refs/tags/*:refs/tags/*'

          echo "Available tags:"
          git tag -l

          if git describe --tags --exact-match >/dev/null 2>&1; then
            echo "Using release version from git tag"
            VERSION=$(git describe --tags --exact-match | sed 's/^v//')
          else
            echo "Using release version from commit hash"
            VERSION="${{ env.HEAD_SHA }}"
          fi

          echo "Using release version: $VERSION"
          echo "$VERSION" > release_version.txt

      - name: Build unit test image
        uses: docker/build-push-action@v6
        with:
          context: .
          target: builder
          tags: builder
          load: true
          cache-from: type=gha,ref=image-cache-${{ github.ref_name }}-${{ matrix.arch }}
          cache-to: type=gha,mode=min,ref=image-cache-${{ github.ref_name }}-${{ matrix.arch }}
          build-args: |
            DEPENDENCY_IMAGE=${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-${{ matrix.arch }}

      - name: Run unit tests
        run: |
          docker run \
            --entrypoint "./silo_test" \
            builder

      - name: Build and push production image
        uses: docker/build-push-action@v6
        with:
          context: .
          push: true
          cache-from: type=gha,ref=image-cache-${{ github.ref_name }}-${{ matrix.arch }}
          cache-to: type=gha,mode=min,ref=image-cache-${{ github.ref_name }}-${{ matrix.arch }}
          tags: ${{ env.DOCKER_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-${{ matrix.arch }}
          build-args: |
            DEPENDENCY_IMAGE=${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-${{ matrix.arch }}

  multiPlatformImages:
    name: Create multi-platform images
    needs: dockerImage
    runs-on: ubuntu-latest
    permissions:
      packages: write
    steps:
      - uses: actions/checkout@v6

      - name: Login to GitHub Container Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Determine HEAD_SHA
        run: |
          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
            echo "HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
          else
            echo "HEAD_SHA=${{ github.sha }}" >> $GITHUB_ENV
          fi

      - name: Determine whether to use a release version
        run: |
          git fetch origin 'refs/tags/*:refs/tags/*'

          echo "Available tags:"
          git tag -l

          if git describe --tags --exact-match >/dev/null 2>&1; then
            echo "Using release version from git tag"
            VERSION=$(git describe --tags --exact-match | sed 's/^v//')
            echo "SILO_VERSION=${VERSION}" >> $GITHUB_ENV
          fi

      - name: Docker metadata
        id: dockerMetadataImage
        uses: docker/metadata-action@v5
        with:
          images: ${{ env.DOCKER_IMAGE_NAME }}
          tags: |
            type=ref,event=pr
            type=raw,value=${{ github.head_ref || github.ref_name }}
            type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}            
            type=raw,value=commit-${{ env.HEAD_SHA }}
            type=semver,value=${{ env.SILO_VERSION }},enable=${{ env.SILO_VERSION != '' }},pattern={{version}}
            type=semver,value=${{ env.SILO_VERSION }},enable=${{ env.SILO_VERSION != '' }},pattern={{major}}.{{minor}}

      - name: Tag images
        run: |
          TAGS=(${{ steps.dockerMetadataImage.outputs.tags }})
          for TAG in "${TAGS[@]}"; do
            docker buildx imagetools create --tag $TAG \
              "${{ env.DOCKER_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-arm64" \
              "${{ env.DOCKER_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-amd64"
          done

  endToEndTests:
    name: Run End To End Tests
    needs: dockerImage
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6

      - name: Use Node.js ${{ matrix.node-version }}
        uses: actions/setup-node@v6
        with:
          node-version: 18.x

      - name: Determine HEAD_SHA
        run: |
          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
            echo "HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
          else
            echo "HEAD_SHA=${{ github.sha }}" >> $GITHUB_ENV
          fi

      - uses: actions/cache@v4
        with:
          path: ~/.npm
          key: "${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}"

      - name: npm install
        run: cd endToEndTests && npm ci

      - name: Check Format
        run: cd endToEndTests && npm run check-format

      - name: Start Docker Container and preprocess data
        run: docker compose -f docker-compose-for-tests-preprocessing-from-ndjson.yml up
        env:
          SILO_IMAGE: ${{ env.DOCKER_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-amd64

      - name: Start Docker Container and run api
        run: docker compose -f docker-compose-for-tests-api.yml up -d --wait
        env:
          SILO_IMAGE: ${{ env.DOCKER_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-amd64

      - name: Run Tests
        run: cd endToEndTests && SILO_URL=localhost:8080 npm run test

  linterChanges:
    name: Build/Run linter on changed files
    needs: dockerImage
    if: github.event_name == 'pull_request'
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - name: Determine HEAD_SHA
        run: |
          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
            echo "HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
          else
            echo "HEAD_SHA=${{ github.sha }}" >> $GITHUB_ENV
          fi

      - name: Run linter script
        run: bash ./linter_changes.bash ghcr.io/genspectrum/lapis-silo-dependencies:commit-${{ env.HEAD_SHA }}-amd64


  linterAll:
    name: Build/Run linter on all files
    needs: dockerImage
    if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'trigger-linter')
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6

      - name: Determine HEAD_SHA
        run: |
          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
            echo "HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
          else
            echo "HEAD_SHA=${{ github.sha }}" >> $GITHUB_ENV
          fi

      - shell: bash
        name: Build all files with clang-tidy
        run: |
          docker build -f ./Dockerfile_linter \
            --build-arg DEPENDENCY_IMAGE=ghcr.io/genspectrum/lapis-silo-dependencies:commit-${{ env.HEAD_SHA }}-amd64 .