genai-pre-merge.yaml

---
# SPDX-FileCopyrightText: (C) 2025 Intel Corporation
# SPDX-License-Identifier: Apache-2.0

name: "[GENAI] PR Scans and Unit Tests"
run-name: "[GENAI] PR  workflow (by @${{ github.actor }} via ${{ github.event_name }})"


# Only run at most 1 workflow concurrently per PR, unlimited for branches
concurrency:
  group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.sha }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

on:
  pull_request:
    branches:
      - main
    paths:
      - 'sample-applications/chat-question-and-answer-core/**'
      - 'sample-applications/chat-question-and-answer/**'
      - 'microservices/document-ingestion/pgvector/**'

jobs:
  zizmor-workflow-scan:
    runs-on: ubuntu-22.04-32core-128GB  
    permissions:
      contents: read
    env:
      ZIZMOR_VERSION: 1.5.2
    steps:
      - name: Checkout Code
        uses: actions/checkout@v4
        with:
          persist-credentials: false
          
      - name: Install uv
        uses: astral-sh/setup-uv@f0ec1fc3b38f5e7cd731bb6ce540c5af426746bb
        
      - name: Run Zizmor Workflow Security Scan
        continue-on-error: true
        run: uvx zizmor=="$ZIZMOR_VERSION" "$GITHUB_WORKSPACE" --no-exit-codes > zizmor_workflow_scan_report.txt
          
      - name: Upload Zizmor Scan Report
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: zizmor-workflow-security-report
          path: zizmor_workflow_scan_report.txt
  
  detect-changes:
    runs-on: ubuntu-22.04-32core-128GB
    permissions:
      contents: read
    outputs:
      changed_projects: ${{ steps.discover.outputs.changed_projects }}
    steps:
      - name: Checkout Code
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          persist-credentials: false

      - name: Discover Changed Subfolders
        id: discover
        uses: ./.github/actions/discover-changed-subfolders
        with:
          project_folder: "."
          include_folders: "sample-applications/chat-question-and-answer-core,sample-applications/chat-question-and-answer,microservices/document-ingestion/pgvector"

      - name: Debug Changed Projects
        shell: bash
        env:
          CHANGED_PROJECTS: ${{ steps.discover.outputs.changed_projects }}
        run: |
          echo "GITHUB_WORKSPACE: $GITHUB_WORKSPACE"
          echo "PWD: $(pwd)"
          echo "Changed Projects: $CHANGED_PROJECTS"
  
  chat-question-and-answer-core-job:
    needs: detect-changes
    permissions:
      contents: read    
    if: contains(fromJson(needs.detect-changes.outputs.changed_projects), 'sample-applications/chat-question-and-answer-core')
    runs-on: ubuntu-22.04-32core-128GB
    env:
      HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACE_API_TOKEN }}
      LLM_MODEL: Intel/neural-chat-7b-v3-3
      EMBEDDING_MODEL_NAME: BAAI/bge-small-en-v1.5
      RERANKER_MODEL: BAAI/bge-reranker-base    
    steps:
      - name: Checkout Code
        uses: actions/checkout@v4
        with:
          persist-credentials: false   
      
      - name: Setup Tools
        uses: ./.github/actions/setup-tools
      
      - name: Run Unit Tests for Core
        continue-on-error: true
        shell: bash
        run: |
          cd sample-applications/chat-question-and-answer-core
          echo "Running unit test cases"
          python3.12 -m venv venv
          source venv/bin/activate
          poetry install --with dev || true
          poetry add pytest-html
          source scripts/setup_env.sh
          poetry run pytest tests/ --html=pytest-coverage.html
          coverage run --source=. -m pytest
          coverage report -m
          # Install required packages
          #poetry add pytest-cov pytest-html
          
          # Run tests with coverage and HTML report in one command
          # poetry run pytest tests/ --cov=. --cov-report=html --html=pytest-report.html
          
          # For a combined coverage and test report
          # poetry run pytest tests/ --cov=. --cov-report=term --cov-report=html:coverage-html --html=pytest-report.html
          deactivate
          rm -rf venv
          
      - name: Upload Coverage Report
        uses: actions/upload-artifact@v4
        continue-on-error: true
        with:
          name: core-coverage-report
          path: |
            sample-applications/chat-question-and-answer-core/pytest-coverage.html
            sample-applications/chat-question-and-answer-core/pytest-report.html
            # sample-applications/chat-question-and-answer-core/htmlcov/index.html
            # sample-applications/chat-question-and-answer-core/coverage-html/index.html
      
      - name: Setup Node.js
        uses: actions/setup-node@v4
        continue-on-error: true
        with:
          node-version: '22'
 
      - name: Install npm dependencies
        continue-on-error: true
        shell: bash
        run: |
          pwd
          cd sample-applications/chat-question-and-answer-core/ui/
          # Install with verbose logging
          npm install
          npm install -D vitest@2.1.9
          npm run test
          npm run test:ui
          npm run coverage
          npx vitest run --reporter=html --outputFile=ui-results.html --coverage --coverage.reporter=html --coverage.reportsDirectory=ui-coverage-html
      
      - name: Upload UI Results
        uses: actions/upload-artifact@v4
        continue-on-error: true
        if: always()
        with:
          name: ui-test-results-core
          path: |
              sample-applications/chat-question-and-answer-core/ui/ui-results.html
              sample-applications/chat-question-and-answer-core/ui/ui-coverage-html/
      
      - name: trivy repo scan
        continue-on-error: true
        shell: bash
        run: |
          pwd
          cd sample-applications/chat-question-and-answer-core/
          trivy --version
          which trivy
          trivy image --download-db-only
          curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/html.tpl -o trivy-html.tpl
    
          # Use the downloaded template
          trivy fs . --format template --template "@trivy-html.tpl" -o "trivy_code_scan_core.html"   
      
      - name: Upload trivy reports
        continue-on-error: true
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: trivy-code-scan-results-core
          path: |
              sample-applications/chat-question-and-answer-core/trivy_code_scan_core.html
  

      - name: ClamAV Antivirus Scan
        continue-on-error: true
        shell: bash
        run: |
          echo "Starting ClamAV scan on sample-applications/chat-question-and-answer-core/..."
          
          docker run --rm \
            --mount type=bind,source=./sample-applications/chat-question-and-answer-core/,target=/scandir \
            clamav/clamav:stable \
            clamscan --recursive --log=/scandir/clamav-scan-report.log \
            /scandir
          
          SCAN_EXIT_CODE=$?
          sudo chown $USER:$USER sample-applications/chat-question-and-answer-core/clamav-scan-report.log 2>/dev/null || true
          
          if [ $SCAN_EXIT_CODE -ne 0 ]; then
            echo "ClamAV scan failed or found issues"
            exit 1
          fi
      
      - name: Upload Antivirus Report
        continue-on-error: true
        if: always()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
        with:
          name: antivirus-report-core
          path: sample-applications/chat-question-and-answer-core/clamav-scan-report.log
      
      - name: Trivy Image Scan
        continue-on-error: true
        shell: bash
        run: |
              pwd
              curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/html.tpl -o trivy-html.tpl
              echo "Building image chatqna-core-backend and scanning"
              docker build -f ./sample-applications/chat-question-and-answer-core/docker/Dockerfile -t chatqna-core-backend:latest ./sample-applications/chat-question-and-answer-core/
              trivy image chatqna-core-backend:latest --ignore-unfixed --format template --template "@trivy-html.tpl" -o sample-applications/chat-question-and-answer-core/trivy_image_scan_core-backend.html
              trivy image --quiet --format spdx-json --output sample-applications/chat-question-and-answer-core/trivy_image_scan_core-backend.spdx.json chatqna-core-backend:latest
              echo "completed chatqna-core-backend scanning"
              
              echo "**************************************************************"
              echo "Building image chatqna-core-frontend and scanning"
              docker build -t="chatqna-core-frontend:latest" ./sample-applications/chat-question-and-answer-core/ui
              trivy image chatqna-core-frontend:latest --ignore-unfixed --format template --template "@trivy-html.tpl" -o sample-applications/chat-question-and-answer-core/trivy_image_scan_core-frontend.html
              trivy image --quiet --format spdx-json --output sample-applications/chat-question-and-answer-core/trivy_image_scan_core-frontend.spdx.json chatqna-core-frontend:latest
              echo "completed chatqna-core-frontend scanning"
              echo "print all the files"
              pwd
              ls -lrt
              echo "**************************************************************"
            
            
      - name: Upload Trivy Image Report
        continue-on-error: true
        if: always()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
        with:
          name: Trivy image scan report-core
          path: |
                sample-applications/chat-question-and-answer-core/trivy_image_scan_core-backend.html
                sample-applications/chat-question-and-answer-core/trivy_image_scan_core-backend.spdx.json
                sample-applications/chat-question-and-answer-core/trivy_image_scan_core-frontend.html
                sample-applications/chat-question-and-answer-core/trivy_image_scan_core-frontend.spdx.json
      - name: Run Bandit Security Scan
        continue-on-error: true
        shell: bash
        run: |
          echo "Running Bandit security scan..."
          
          python -m bandit -r sample-applications/chat-question-and-answer-core/ -v --exit-zero > bandit_scan_report_summary.txt || echo "Bandit found security issues"
          
          echo "Bandit scan completed"
      
      - name: Upload Bandit Security Report
        if: always()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
        with:
          name: bandit-security-report-core
          path: |
            bandit_scan_report_summary.txt  
      
      
  chat-question-and-answer-job:
    needs: detect-changes
    permissions:
      contents: read    
    if: contains(fromJson(needs.detect-changes.outputs.changed_projects), 'sample-applications/chat-question-and-answer')
    runs-on: ubuntu-22.04-32core-128GB
    env:
      HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACE_API_TOKEN }}
      LLM_MODEL: Intel/neural-chat-7b-v3-3
      EMBEDDING_MODEL_NAME: BAAI/bge-small-en-v1.5
      RERANKER_MODEL: BAAI/bge-reranker-base    
    steps:
      - name: Checkout Code
        uses: actions/checkout@v4
        with:
          persist-credentials: false
      
      - name: Setup Tools
        uses: ./.github/actions/setup-tools
      
      - name: Run Unit Tests for Core
        continue-on-error: true
        shell: bash
        run: |
          cd sample-applications/chat-question-and-answer
          echo "Running unit test cases"
          python3.12 -m venv venv
          source venv/bin/activate
          poetry install --with dev || true
          #poetry add pytest-html
          #source setup.sh
          source setup.sh llm=TGI embed=TEI
          poetry run pytest tests/unit_tests/ --html=pytest-coverage-chatqna.html
          coverage run --source=. -m pytest
          coverage report -m
          # Install required packages
          # poetry add pytest-cov pytest-html 
          
          # Run tests with coverage and HTML report in one command
          #poetry run pytest tests/unit_tests/ --cov=. --cov-report=html --html=pytest-report-chatqna.html
          
          # For a combined coverage and test report
          # poetry run pytest tests/unit_tests/ --cov=. --cov-report=term --cov-report=html:coverage-html --html=pytest-report-chatqna.html     
          deactivate
          rm -rf venv
          
      - name: Upload Coverage Report
        continue-on-error: true
        uses: actions/upload-artifact@v4
        with:
          name: chatqna-coverage-report-qa
          path: |
            sample-applications/chat-question-and-answer/pytest-coverage-chatqna.html
            sample-applications/chat-question-and-answer/pytest-report-chatqna.html
            # sample-applications/chat-question-and-answer/htmlcov/index.html
            # sample-applications/chat-question-and-answer/coverage-html/index.html
      
      # - name: Setup Node.js
      #   uses: actions/setup-node@v4
      #   with:
      #     node-version: '22'
 
      - name: Install npm dependencies
        continue-on-error: true
        shell: bash
        run: |
          pwd
          cd sample-applications/chat-question-and-answer/ui/react
          npm install
          npm install -D vitest@2.1.9
          npm run test
          npm run test:ui
          npm run coverage
          npx vitest run --reporter=html --outputFile=chatqna-ui-results.html --coverage --coverage.reporter=html --coverage.reportsDirectory=chat-qna-ui-coverage-html
      
      - name: Upload UI Results
        continue-on-error: true
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: chatqna-ui-test-results-qa
          path: |
              sample-applications/chat-question-and-answer/ui/react/ui-results.html
              sample-applications/chat-question-and-answer/ui/react/ui-coverage-html/
      - name: trivy repo scan
        continue-on-error: true
        shell: bash
        run: |
          pwd
          cd sample-applications/chat-question-and-answer/
          trivy --version
          trivy image --download-db-only
          curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/html.tpl -o trivy-html.tpl
    
          # Use the downloaded template
          trivy fs . --format template --template "@trivy-html.tpl" -o "trivy_code_scan_chatqna.html"   
      
      - name: Upload trivy reports
        continue-on-error: true
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: trivy-code-scan-results-chatqna
          path: |
              sample-applications/chat-question-and-answer/trivy_code_scan_chatqna.html
      - name: ClamAV Antivirus Scan
        continue-on-error: true
        shell: bash
        run: |
          echo "Starting ClamAV scan on sample-applications/chat-question-and-answer/..."
          
          docker run --rm \
            --mount type=bind,source=./sample-applications/chat-question-and-answer/,target=/scandir \
            clamav/clamav:stable \
            clamscan --recursive --log=/scandir/clamav-scan-report.log \
            /scandir
          
          SCAN_EXIT_CODE=$?
          sudo chown $USER:$USER sample-applications/chat-question-and-answer/clamav-scan-report.log 2>/dev/null || true
          
          if [ $SCAN_EXIT_CODE -ne 0 ]; then
            echo "ClamAV scan failed or found issues"
            exit 1
          fi
      
      - name: Upload Antivirus Report
        continue-on-error: true
        if: always()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
        with:
          name: antivirus-report-qa
          path: sample-applications/chat-question-and-answer/clamav-scan-report.log
      
      - name: Trivy Image Scan
        continue-on-error: true
        shell: bash
        run: |
              pwd
              curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/html.tpl -o trivy-html.tpl
              echo "Building image chatqna-backend and scanning"
              docker build -f ./sample-applications/chat-question-and-answer/Dockerfile -t chatqna-backend:latest ./sample-applications/chat-question-and-answer/
              trivy image chatqna-backend:latest --ignore-unfixed --format template --template "@trivy-html.tpl" -o sample-applications/chat-question-and-answer/trivy_image_scan_chatqna_backend.html
              trivy image --quiet --format spdx-json --output sample-applications/chat-question-and-answer/trivy_image_scan_chatqna_backend.spdx.json chatqna-backend:latest
              echo "completed chatqna-backend scanning"
              
              echo "**************************************************************"
              echo "Building image chatqna-frontend and scanning"
              docker build -t="chatqna-frontend:latest" ./sample-applications/chat-question-and-answer/ui/react
              trivy image chatqna-frontend:latest --ignore-unfixed --format template --template "@trivy-html.tpl" -o sample-applications/chat-question-and-answer/trivy_image_scan_chatqna_frontend.html
              trivy image --quiet --format spdx-json --output sample-applications/chat-question-and-answer/trivy_image_scan_chatqna_frontend.spdx.json chatqna-frontend:latest
              echo "completed chatqna-frontend scanning"
              echo "print all the files"
              pwd
              ls -lrt
              echo "**************************************************************"
            
            
      - name: Upload Trivy Image Report
        continue-on-error: true
        if: always()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
        with:
          name: Trivy image scan report-qa
          path: |
                sample-applications/chat-question-and-answer/trivy_image_scan_chatqna_backend.html
                sample-applications/chat-question-and-answer/trivy_image_scan_chatqna_backend.spdx.json
                sample-applications/chat-question-and-answer/trivy_image_scan_chatqna_frontend.html
                sample-applications/chat-question-and-answer/trivy_image_scan_chatqna_frontend.spdx.json
      - name: Run Bandit Security Scan
        continue-on-error: true
        shell: bash
        run: |
          echo "Running Bandit security scan..."
          
          python -m bandit -r sample-applications/chat-question-and-answer/ -v --exit-zero > bandit_scan_report_summary_chatqna.txt || echo "Bandit found security issues"
          
          echo "Bandit scan completed"
      
      - name: Upload Bandit Security Report
        continue-on-error: true
        if: always()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
        with:
          name: bandit-security-report-qa
          path: |
            bandit_scan_report_summary_chatqna.txt
  
  document-ingestion-job:
    needs: detect-changes
    permissions:
      contents: read
    if: contains(fromJson(needs.detect-changes.outputs.changed_projects), 'microservices/document-ingestion/pgvector')
    runs-on: ubuntu-22.04-32core-128GB
    env:
      HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACE_API_TOKEN }}   
    steps:
      - name: Checkout Code
        uses: actions/checkout@v4
        with:
          persist-credentials: false   
      
      - name: Setup Tools
        uses: ./.github/actions/setup-tools
      
      - name: Run Unit Tests for Core
        continue-on-error: true
        shell: bash
        run: |
          cd microservices/document-ingestion/pgvector/
          echo "Running unit test cases"
          python3.12 -m venv venv
          source venv/bin/activate
          poetry install --with dev || true
          poetry add pytest-html
          source run.sh --nosetup
          poetry run pytest tests/unit_tests/ --html=pytest-coverage-doc_ing.html
          coverage run --source=. -m pytest
          coverage report -m
          # Install required packages
          #poetry add pytest-cov pytest-html 
          
          # Run tests with coverage and HTML report in one command
          #poetry run pytest tests/unit_tests/ --cov=. --cov-report=html --html=pytest-report-doc_ing.html
          
          # For a combined coverage and test report
          #poetry run pytest tests/unit_tests/ --cov=. --cov-report=term --cov-report=html:coverage-html --html=pytest-report-doc_ing.html     
          deactivate
          rm -rf venv
          
      - name: Upload Coverage Report
        continue-on-error: true
        uses: actions/upload-artifact@v4
        with:
          name: doc_ing-coverage-report
          path: |
            microservices/document-ingestion/pgvector/pytest-coverage-doc_ing.html
            microservices/document-ingestion/pgvector/pytest-report-doc_ing.html
            # microservices/document-ingestion/pgvector/htmlcov/index.html
            # microservices/document-ingestion/pgvector/coverage-html/index.html
      
     
      - name: trivy repo scan
        continue-on-error: true
        shell: bash
        run: |
          pwd
          cd microservices/document-ingestion/pgvector/
          trivy --version
          trivy image --download-db-only
          curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/html.tpl -o trivy-html.tpl
    
          # Use the downloaded template
          trivy fs . --format template --template "@trivy-html.tpl" -o "trivy_code_scan_doc_ing.html"   
      
      - name: Upload trivy reports
        continue-on-error: true
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: trivy-code-scan-results-doc_ing
          path: |
              microservices/document-ingestion/pgvector/trivy_code_scan_doc_ing.html
      
      - name: ClamAV Antivirus Scan
        continue-on-error: true
        shell: bash
        run: |
          echo "Starting ClamAV scan on sample-applications/chat-question-and-answer/..."
          
          docker run --rm \
            --mount type=bind,source=./microservices/document-ingestion/pgvector/,target=/scandir \
            clamav/clamav:stable \
            clamscan --recursive --log=/scandir/clamav-scan-report.log \
            /scandir
          
          SCAN_EXIT_CODE=$?
          sudo chown $USER:$USER microservices/document-ingestion/pgvector/clamav-scan-report.log 2>/dev/null || true
          
          if [ $SCAN_EXIT_CODE -ne 0 ]; then
            echo "ClamAV scan failed or found issues"
            exit 1
          fi
      
      - name: Upload Antivirus Report
        continue-on-error: true
        if: always()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
        with:
          name: antivirus-report-doc
          path: microservices/document-ingestion/pgvector/clamav-scan-report.log
      
      - name: Trivy Image Scan
        continue-on-error: true
        shell: bash
        run: |
              echo "print pwd"
              curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/html.tpl -o trivy-html.tpl
              echo "Building image doc_ing-backend and scanning"
              docker build -f ./microservices/document-ingestion/pgvector/docker/Dockerfile -t doc_ing-backend:latest ./microservices/document-ingestion/pgvector/
              trivy image doc_ing-backend:latest --ignore-unfixed --format template --template "@trivy-html.tpl" -o microservices/document-ingestion/pgvector/trivy_image_scan_doc_ing.html
              trivy image --quiet --format spdx-json --output microservices/document-ingestion/pgvector/trivy_image_scan_doc_ing.spdx.json doc_ing-backend:latest
              echo "completed doc_ing-backend scanning"
            
              
            
      - name: Upload Trivy Image Report
        continue-on-error: true
        if: always()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
        with:
          name: Trivy image scan report-doc
          path: |
                microservices/document-ingestion/pgvector/trivy_image_scan_doc_ing.html
                microservices/document-ingestion/pgvector/trivy_image_scan_doc_ing.spdx.json
      
      - name: Run Bandit Security Scan
        continue-on-error: true
        shell: bash
        run: |
          echo "Running Bandit security scan..."
          
          python -m bandit -r microservices/document-ingestion/ -v --exit-zero > bandit_scan_report_summary_doc_ing.txt || echo "Bandit found security issues"
          
          echo "Bandit scan completed"
      
      - name: Upload Bandit Security Report
        continue-on-error: true
        if: always()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
        with:
          name: bandit-security-report-doc
          path: |
            bandit_scan_report_summary_doc_ing.txt