Add Enterprise Catalog tests #404
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| branches: | |
| - main | |
| jobs: | |
| discover-tests: | |
| name: Discover Tests | |
| runs-on: ubuntu-latest | |
| outputs: | |
| test_dirs: ${{ steps.discover-tests.outputs.test_dirs }} | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v4 | |
| - name: Discover Test Directories | |
| id: discover-tests | |
| run: | | |
| test_dirs=$(find tests/ -type f \( -name 'test_*.py' -o -name '*_test.py' \) -exec dirname {} \; | sort -u | jq -R -s -c 'split("\n") | map(select(length > 0))') | |
| echo "test_dirs=$test_dirs" >> $GITHUB_OUTPUT | |
| run-parallel-tests: | |
| name: Run Parallel Tests | |
| runs-on: ubuntu-latest | |
| continue-on-error: true | |
| needs: discover-tests | |
| env: | |
| RETRY_COUNT: 12 # number of retries for health checks | |
| SLEEP_INTERVAL: 5 # Sleep duration in seconds between retries | |
| MINIO_HOST_URL: http://localhost:9000 | |
| MINIO_HEALTH_URL: http://localhost:9000/minio/health/live | |
| DREMIO_HEALTH_URL: http://localhost:9047 | |
| MINIO_ROOT_USER: admin | |
| MINIO_ROOT_PASSWORD: password | |
| DREMIO_SOFTWARE_USERNAME: dremio | |
| DREMIO_SOFTWARE_PASSWORD: dremio123 | |
| DREMIO_SOFTWARE_HOST: localhost | |
| DREMIO_DATALAKE: dbt_test_source | |
| DREMIO_DATABASE: dbt_test | |
| DBT_TEST_USER_1: dbt_test_user_1 | |
| DBT_TEST_USER_2: dbt_test_user_2 | |
| DBT_TEST_USER_3: dbt_test_user_3 | |
| strategy: | |
| matrix: | |
| test_dir: ${{ fromJson(needs.discover-tests.outputs.test_dirs) }} | |
| services: | |
| minio: | |
| image: minio/minio:edge-cicd | |
| options: --network-alias minio | |
| ports: | |
| - 9000:9000 | |
| - 9001:9001 | |
| env: | |
| MINIO_ROOT_USER: ${{ env.MINIO_ROOT_USER }} | |
| MINIO_ROOT_PASSWORD: ${{ env.MINIO_ROOT_PASSWORD }} | |
| dremio: | |
| image: dremio/dremio-oss | |
| options: --network-alias dremio | |
| ports: | |
| - 31010:31010 | |
| - 9047:9047 | |
| env: | |
| DREMIO_JAVA_SERVER_EXTRA_OPTS: -Ddebug.addDefaultUser=true | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v4 | |
| - name: Install MinIO Client (mc) | |
| run: bash .github/scripts/install_minio_client.sh | |
| - name: Create MinIO bucket | |
| run: bash .github/scripts/create_minio_bucket.sh | |
| - name: Extract Auth Token | |
| run: bash .github/scripts/extract_auth_token.sh | |
| - name: Create and Format Sources | |
| run: bash .github/scripts/create_and_format_sources.sh | |
| - name: Install Dependencies | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.9' | |
| - name: Install Python Dependencies | |
| run: | | |
| pip install --upgrade pip | |
| pip install -r dev_requirements.txt | |
| pip install . | |
| - name: Create dbt projects | |
| run: bash .github/scripts/create_dbt_projects.sh | |
| - name: Clean up __pycache__ directories | |
| run: | | |
| find . -type d -name "__pycache__" -exec rm -r {} + | |
| - name: Create .env file for tests | |
| run: bash .github/scripts/create_env_file.sh | |
| - name: Sanitize test directory name | |
| id: sanitize | |
| run: echo "sanitized_test_dir=$(echo ${{ matrix.test_dir }} | tr '/' '_')" >> $GITHUB_ENV | |
| - name: Run tests | |
| id: run_tests | |
| run: | | |
| mkdir reports/ | |
| report_file="reports/${{ env.sanitized_test_dir }}.txt" | |
| pytest ${{ matrix.test_dir }} | tee $report_file | |
| exit ${PIPESTATUS[0]} | |
| - name: Upload test report as artifact | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ env.sanitized_test_dir }} | |
| path: reports/ | |
| verify-failures: | |
| name: Verify Expected Test Failures | |
| runs-on: ubuntu-latest | |
| needs: run-parallel-tests | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v3.5.2 | |
| - name: Download All Test Reports | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: reports/ | |
| - name: Compare Actual Failures with Expected Failures | |
| run: bash .github/scripts/compare_test_failures.sh |