feat(silo): add codeql workflow #3729
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: LAPIS-SILO | |
| on: | |
| pull_request: | |
| workflow_run: | |
| workflows: [Release SILO] | |
| types: [completed] | |
| branches: [main] | |
| env: | |
| DOCKER_DEPENDENCY_IMAGE_NAME: ghcr.io/genspectrum/lapis-silo-dependencies | |
| DOCKER_IMAGE_NAME: ghcr.io/genspectrum/lapis-silo | |
| jobs: | |
| dockerImage: | |
| name: Build docker images | |
| strategy: | |
| matrix: | |
| arch: ["amd64", "arm64"] | |
| runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-22.04-arm' || 'ubuntu-latest' }} | |
| permissions: | |
| packages: write | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Determine HEAD_SHA | |
| run: | | |
| if [[ "${{ github.event_name }}" == "pull_request" ]]; then | |
| echo "HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV | |
| else | |
| echo "HEAD_SHA=${{ github.sha }}" >> $GITHUB_ENV | |
| fi | |
| - name: Generate dependency files hash | |
| run: | | |
| DIR_HASH=$(echo -n ${{ hashFiles('conanfile.py', 'conanprofile.docker', './Dockerfile_dependencies') }}) | |
| echo "DIR_HASH=$DIR_HASH" >> $GITHUB_ENV | |
| - name: Check if dependency image exists | |
| run: | | |
| EXISTS=$(docker manifest inspect \ | |
| ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:filehash-${{ env.DIR_HASH }}-${{ matrix.arch }} > /dev/null 2>&1 && | |
| echo "true" || echo "false") | |
| echo "CACHE_HIT=$EXISTS" >> $GITHUB_ENV | |
| - name: Build and push dependency image if input files changed | |
| if: env.CACHE_HIT == 'false' | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| file: Dockerfile_dependencies | |
| push: true | |
| tags: ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:filehash-${{ env.DIR_HASH }}-${{ matrix.arch }} | |
| cache-from: type=gha,ref=dependency-image-cache-${{ matrix.arch }}-${{ hashFiles('conanfile.py', 'Dockerfile') }} | |
| cache-to: type=gha,mode=min,ref=dependency-image-cache-${{ hashFiles('conanfile.py', 'Dockerfile') }} | |
| - name: Tag dependency image with commit hash | |
| run: | | |
| docker buildx imagetools create \ | |
| --tag ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-${{ matrix.arch }} \ | |
| ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:filehash-${{ env.DIR_HASH }}-${{ matrix.arch }} | |
| - name: Write release_version.txt | |
| run: | | |
| git fetch origin 'refs/tags/*:refs/tags/*' | |
| echo "Available tags:" | |
| git tag -l | |
| if git describe --tags --exact-match >/dev/null 2>&1; then | |
| echo "Using release version from git tag" | |
| VERSION=$(git describe --tags --exact-match | sed 's/^v//') | |
| else | |
| echo "Using release version from commit hash" | |
| VERSION="${{ env.HEAD_SHA }}" | |
| fi | |
| echo "Using release version: $VERSION" | |
| echo "$VERSION" > release_version.txt | |
| - name: Build unit test image | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| target: builder | |
| tags: builder | |
| load: true | |
| cache-from: type=gha,ref=image-cache-${{ github.ref_name }}-${{ matrix.arch }} | |
| cache-to: type=gha,mode=min,ref=image-cache-${{ github.ref_name }}-${{ matrix.arch }} | |
| build-args: | | |
| DEPENDENCY_IMAGE=${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-${{ matrix.arch }} | |
| - name: Run unit tests | |
| run: | | |
| docker run \ | |
| --entrypoint "./silo_test" \ | |
| builder | |
| - name: Build and push production image | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| push: true | |
| cache-from: type=gha,ref=image-cache-${{ github.ref_name }}-${{ matrix.arch }} | |
| cache-to: type=gha,mode=min,ref=image-cache-${{ github.ref_name }}-${{ matrix.arch }} | |
| tags: ${{ env.DOCKER_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-${{ matrix.arch }} | |
| build-args: | | |
| DEPENDENCY_IMAGE=${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-${{ matrix.arch }} | |
| - name: Run clang-tidy inside builder | |
| run: | | |
| docker run --rm \ | |
| -v "$GITHUB_WORKSPACE:/src" \ | |
| -w /src \ | |
| builder \ | |
| bash -eo pipefail -c ' | |
| # Install tools needed for the analysis (kept inside container) | |
| if command -v apt-get >/dev/null 2>&1; then | |
| apt-get update -y | |
| # clang-tidy + helpers. jq used to read compile_commands.json | |
| apt-get install -y --no-install-recommends clang clang-tidy jq python3 | |
| fi | |
| # 2a) Ensure deps are generated (your Makefile would do this, but do it here explicitly) | |
| conan --version >/dev/null 2>&1 || { echo "Conan missing in builder image"; exit 1; } | |
| # Generate CMake files and compile_commands.json (explicitly set export flag) | |
| cmake -S . -B build/Debug \ | |
| -D CMAKE_BUILD_TYPE=Debug \ | |
| -D CMAKE_EXPORT_COMPILE_COMMANDS=ON | |
| # Optional: build to materialize any generated headers if your project needs them | |
| cmake --build build/Debug -j "$(nproc)" | |
| # Sanity check | |
| test -f build/Debug/compile_commands.json || { echo "compile_commands.json not found"; exit 2; } | |
| # Choose checks. Start strict but practical. | |
| CHECKS="-*,bugprone-*,clang-analyzer-*,cppcoreguidelines-*,cert-*,performance-*,readability-*" | |
| # Treat every diagnostic as error to block merges | |
| WARNERR="*" | |
| # Run clang-tidy over all TU paths from the compile database. | |
| # This avoids missing files and respects per-target flags. | |
| FILES=$(jq -r ".[].file" build/Debug/compile_commands.json | sort -u) | |
| if [ -z "$FILES" ]; then | |
| echo "No files in compile_commands.json"; exit 3; | |
| fi | |
| # Use parallelism; fall back if run-clang-tidy isn’t present. | |
| if command -v run-clang-tidy >/dev/null 2>&1; then | |
| run-clang-tidy \ | |
| -p build/Debug \ | |
| -j "$(nproc)" \ | |
| -header-filter="^src/|^include/" \ | |
| -checks="$CHECKS" \ | |
| -warnings-as-errors="$WARNERR" | |
| else | |
| # Manual loop if run-clang-tidy isn’t available | |
| failed=0 | |
| for f in $FILES; do | |
| echo "=== clang-tidy: $f ===" | |
| clang-tidy "$f" \ | |
| -p build/Debug \ | |
| -checks="$CHECKS" \ | |
| -warnings-as-errors="$WARNERR" \ | |
| || failed=1 | |
| done | |
| exit $failed | |
| fi | |
| ' | |
| multiPlatformImages: | |
| name: Create multi-platform images | |
| needs: dockerImage | |
| runs-on: ubuntu-latest | |
| permissions: | |
| packages: write | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Determine HEAD_SHA | |
| run: | | |
| if [[ "${{ github.event_name }}" == "pull_request" ]]; then | |
| echo "HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV | |
| else | |
| echo "HEAD_SHA=${{ github.sha }}" >> $GITHUB_ENV | |
| fi | |
| - name: Determine whether to use a release version | |
| run: | | |
| git fetch origin 'refs/tags/*:refs/tags/*' | |
| echo "Available tags:" | |
| git tag -l | |
| if git describe --tags --exact-match >/dev/null 2>&1; then | |
| echo "Using release version from git tag" | |
| VERSION=$(git describe --tags --exact-match | sed 's/^v//') | |
| echo "SILO_VERSION=${VERSION}" >> $GITHUB_ENV | |
| fi | |
| - name: Docker metadata | |
| id: dockerMetadataImage | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: ${{ env.DOCKER_IMAGE_NAME }} | |
| tags: | | |
| type=ref,event=pr | |
| type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }} | |
| type=raw,value=commit-${{ env.HEAD_SHA }} | |
| type=semver,value=${{ env.SILO_VERSION }},enable=${{ env.SILO_VERSION != '' }},pattern={{version}} | |
| type=semver,value=${{ env.SILO_VERSION }},enable=${{ env.SILO_VERSION != '' }},pattern={{major}}.{{minor}} | |
| - name: Tag images | |
| run: | | |
| TAGS=(${{ steps.dockerMetadataImage.outputs.tags }}) | |
| for TAG in "${TAGS[@]}"; do | |
| docker buildx imagetools create --tag $TAG \ | |
| "${{ env.DOCKER_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-arm64" \ | |
| "${{ env.DOCKER_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-amd64" | |
| done | |
| endToEndTests: | |
| name: Run End To End Tests | |
| needs: dockerImage | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Use Node.js ${{ matrix.node-version }} | |
| uses: actions/setup-node@v5 | |
| with: | |
| node-version: 18.x | |
| - name: Determine HEAD_SHA | |
| run: | | |
| if [[ "${{ github.event_name }}" == "pull_request" ]]; then | |
| echo "HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV | |
| else | |
| echo "HEAD_SHA=${{ github.sha }}" >> $GITHUB_ENV | |
| fi | |
| - uses: actions/cache@v4 | |
| with: | |
| path: ~/.npm | |
| key: "${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}" | |
| - name: npm install | |
| run: cd endToEndTests && npm ci | |
| - name: Check Format | |
| run: cd endToEndTests && npm run check-format | |
| - name: Start Docker Container and preprocess data | |
| run: docker compose -f docker-compose-for-tests-preprocessing-from-ndjson.yml up | |
| env: | |
| SILO_IMAGE: ${{ env.DOCKER_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-amd64 | |
| - name: Start Docker Container and run api | |
| run: docker compose -f docker-compose-for-tests-api.yml up -d --wait | |
| env: | |
| SILO_IMAGE: ${{ env.DOCKER_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-amd64 | |
| - name: Run Tests | |
| run: cd endToEndTests && SILO_URL=localhost:8080 npm run test | |
| linterChanges: | |
| name: Build/Run linter on changed files | |
| needs: dockerImage | |
| if: github.event_name == 'pull_request' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v5 | |
| with: | |
| fetch-depth: 0 | |
| - name: Determine HEAD_SHA | |
| run: | | |
| if [[ "${{ github.event_name }}" == "pull_request" ]]; then | |
| echo "HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV | |
| else | |
| echo "HEAD_SHA=${{ github.sha }}" >> $GITHUB_ENV | |
| fi | |
| - name: Run linter script | |
| run: bash ./linter_changes.bash ghcr.io/genspectrum/lapis-silo-dependencies:commit-${{ env.HEAD_SHA }}-amd64 | |
| linterAll: | |
| name: Build/Run linter on all files | |
| needs: dockerImage | |
| if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'trigger-linter') | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Determine HEAD_SHA | |
| run: | | |
| if [[ "${{ github.event_name }}" == "pull_request" ]]; then | |
| echo "HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV | |
| else | |
| echo "HEAD_SHA=${{ github.sha }}" >> $GITHUB_ENV | |
| fi | |
| - shell: bash | |
| name: Build all files with clang-tidy | |
| run: | | |
| docker build -f ./Dockerfile_linter \ | |
| --build-arg DEPENDENCY_IMAGE=ghcr.io/genspectrum/lapis-silo-dependencies:commit-${{ env.HEAD_SHA }}-amd64 . |