test.yml

name: Test

on:
  push:
    branches: 
      - master
      - develop
    paths-ignore:
      - "docs/**"
  pull_request:
    branches:
      - master
      - develop
    paths-ignore:
      - "docs/**"
jobs:
  unit_tests:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: [ '3.8', '3.9', '3.10', '3.11']

    steps:
    - uses: actions/checkout@v4
    
    - name: Setup python ${{ matrix.python-version }}
      uses: actions/setup-python@v4.7.1
      with:
        python-version: ${{ matrix.python-version }}

    - name: Setup poetry
      run: |
        python -V
        python -m pip install poetry

    - name: Check pre-commit status
      run: |
        poetry install -v
        poetry run pre-commit run --all-files

    - name: Test with tox
      run: |
        pip install tox-pip-version tox-gh-actions "tox<4.0.0"
        tox -v

    - name: Store coverage reports
      uses: actions/upload-artifact@v4
      with:
        name: coverage-${{ matrix.python-version }}
        path: coverage.xml
        if-no-files-found: error

  sonarcloud:
    runs-on: ubuntu-latest
    needs: unit_tests
    steps:

    - uses: actions/checkout@v2
      with:
        fetch-depth: 0

    - uses: actions/download-artifact@v4
      with:
        name: coverage-3.9
        path: .

    - name: SonarCloud Scan
      uses: sonarsource/sonarcloud-github-action@master
      with:
        projectBaseDir: ${{ github.workspace }}
      env:
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        SONAR_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }}

  codeql:
    runs-on: ubuntu-latest
    needs: unit_tests
    permissions:
      actions: read
      contents: read
      security-events: write

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      # Initializes the CodeQL tools for scanning.
      - name: Initialize CodeQL
        uses: github/codeql-action/init@v2
        with:
          languages: python

      - name: Autobuild
        uses: github/codeql-action/autobuild@v2

      - name: Perform CodeQL Analysis
        uses: github/codeql-action/analyze@v2

  e2e_tests:
    if: (github.event.pull_request.head.repo.full_name || github.push.repository.full_name) == github.repository && false # disabling e2e since env is down
    runs-on: ubuntu-latest
    needs: [unit_tests, sonarcloud, codeql]
    steps:
      - uses: actions/checkout@v2

      - name: Setup python
        uses: actions/setup-python@v4.7.1
        with:
          python-version: '3.10'

      - name: Setup virtualenv
        run: |
          python -V
          python -m pip install virtualenv
          virtualenv venv
          source venv/bin/activate

      - name: Initialize kedro spaceflights project
        run: |
          pip install .
          kedro new --starter spaceflights-pandas --config tests/e2e/starter-config.yml --verbose

      - name: Install project dependencies
        working-directory: ./spaceflights
        run: |
          echo "kedro-kubeflow[gcp] @ git+https://github.com/getindata/kedro-kubeflow.git@$GITHUB_SHA" >> src/requirements.txt
          echo "kedro-docker" >> src/requirements.txt
          sed -i '/kedro-telemetry/d' src/requirements.txt
          cat src/requirements.txt
          pip install -r src/requirements.txt

      - name: Init and update configuration
        working-directory: ./spaceflights
        run: |
          sed -i "s/\(CONFIG_LOADER_ARGS.*\)$/from omegaconf.resolvers import oc\n\1\n      \"custom_resolvers\": { \"oc.env\": oc.env },/g" src/spaceflights/settings.py
          kedro docker init
          kedro kubeflow init http://localhost:9000
          echo "!data/01_raw" >> .dockerignore
          cp ../tests/e2e/catalog.yml conf/base/catalog.yml
          rm -f conf/base/kubeflow.y*
          cp ../tests/e2e/kubeflow.yml conf/base/kubeflow.yml
          

      - name: Prepare docker env
        uses: docker/setup-buildx-action@v3
        with:
          install: true

      - name: Build pipeline docker image
        working-directory: ./spaceflights
        run: |
          docker build --build-arg BASE_IMAGE=python:3.8-buster --tag kedro-kubeflow-e2e:latest --load .

      - name: Publish docker image to GCR
        uses: mattes/gce-docker-push-action@v1
        with:
          creds: ${{ secrets.GOOGLE_CLOUD_TOKEN_JSON }}
          src: kedro-kubeflow-e2e:latest
          dst: gcr.io/gid-ml-ops-sandbox/kedro-kubeflow-e2e:${{ github.sha }}

      - name: Set up GCP Credentials
        uses: google-github-actions/auth@v1.1.1
        with:
          credentials_json: ${{ secrets.GOOGLE_CLOUD_TOKEN_JSON }}
          create_credentials_file: true
          cleanup_credentials: true

      - name: Run project on kubeflow pipelines
        working-directory: spaceflights
        run: |
          kedro kubeflow upload-pipeline
          kedro kubeflow run-once --wait-for-completion --timeout 1200
        env:
          KEDRO_CONFIG_KUBEFLOW_HOSTNAME: ${{ secrets.KUBEFLOW_HOSTNAME }}
          KEDRO_CONFIG_COMMIT_ID: ${{ github.sha }}
          KEDRO_CONFIG_RUN_ID: ${{ github.run_id }}