dbt 1.9 migration #336
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| branches: | |
| - main | |
| jobs: | |
| discover-tests: | |
| name: Discover Tests | |
| runs-on: ubuntu-latest | |
| outputs: | |
| test_dirs: ${{ steps.discover-tests.outputs.test_dirs }} | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v4 | |
| - name: Discover Test Directories | |
| id: discover-tests | |
| run: | | |
| test_dirs=$(find tests/ -type f \( -name 'test_*.py' -o -name '*_test.py' \) -exec dirname {} \; | sort -u | jq -R -s -c 'split("\n") | map(select(length > 0))') | |
| echo "test_dirs=$test_dirs" >> $GITHUB_OUTPUT | |
| run-parallel-tests: | |
| name: Run Parallel Tests | |
| runs-on: ubuntu-latest | |
| needs: discover-tests | |
| env: | |
| RETRY_COUNT: 12 # number of retries for health checks | |
| SLEEP_INTERVAL: 5 # Sleep duration in seconds between retries | |
| MINIO_HOST_URL: http://localhost:9000 | |
| MINIO_HEALTH_URL: http://localhost:9000/minio/health/live | |
| DREMIO_HEALTH_URL: http://localhost:9047 | |
| MINIO_ROOT_USER: admin | |
| MINIO_ROOT_PASSWORD: password | |
| DREMIO_SOFTWARE_USERNAME: dremio | |
| DREMIO_SOFTWARE_PASSWORD: dremio123 | |
| DREMIO_SOFTWARE_HOST: localhost | |
| DREMIO_DATALAKE: dbt_test_source | |
| DREMIO_DATABASE: dbt_test | |
| DBT_TEST_USER_1: dbt_test_user_1 | |
| DBT_TEST_USER_2: dbt_test_user_2 | |
| DBT_TEST_USER_3: dbt_test_user_3 | |
| strategy: | |
| matrix: | |
| test_dir: ${{ fromJson(needs.discover-tests.outputs.test_dirs) }} | |
| services: | |
| minio: | |
| image: minio/minio:edge-cicd | |
| options: --network-alias minio | |
| ports: | |
| - 9000:9000 | |
| - 9001:9001 | |
| env: | |
| MINIO_ROOT_USER: ${{ env.MINIO_ROOT_USER }} | |
| MINIO_ROOT_PASSWORD: ${{ env.MINIO_ROOT_PASSWORD }} | |
| dremio: | |
| image: dremio/dremio-oss | |
| options: --network-alias dremio | |
| ports: | |
| - 31010:31010 | |
| - 9047:9047 | |
| env: | |
| DREMIO_JAVA_SERVER_EXTRA_OPTS: -Ddebug.addDefaultUser=true | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v4 | |
| - name: Install MinIO Client (mc) | |
| run: bash .github/scripts/install_minio_client.sh | |
| - name: Create MinIO bucket | |
| run: bash .github/scripts/create_minio_bucket.sh | |
| - name: Create and Format Sources | |
| run: bash .github/scripts/create_and_format_sources.sh | |
| - name: Install Dependencies | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.9' | |
| - name: Install Python Dependencies | |
| run: | | |
| pip install --upgrade pip | |
| pip install -r dev_requirements.txt | |
| pip install . | |
| - name: Create dbt test users | |
| run: bash .github/scripts/create_dbt_test_users.sh | |
| - name: Create dbt projects | |
| run: bash .github/scripts/create_dbt_projects.sh | |
| - name: Clean up __pycache__ directories | |
| run: | | |
| find . -type d -name "__pycache__" -exec rm -r {} + | |
| - name: Create .env file for tests | |
| run: bash .github/scripts/create_env_file.sh | |
| - name: Sanitize test directory name | |
| id: sanitize | |
| run: echo "sanitized_test_dir=$(echo ${{ matrix.test_dir }} | tr '/' '_')" >> $GITHUB_ENV | |
| - name: Run tests | |
| continue-on-error: true | |
| run: | | |
| mkdir reports/ | |
| report_file="reports/${{ env.sanitized_test_dir }}.txt" | |
| pytest ${{ matrix.test_dir }} | tee $report_file | |
| - name: Upload test report as artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ env.sanitized_test_dir }} | |
| path: reports/ | |
| verify-failures: | |
| name: Verify Expected Test Failures | |
| runs-on: ubuntu-latest | |
| needs: run-parallel-tests | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v3.5.2 | |
| - name: Download All Test Reports | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: reports/ | |
| - name: Compare Actual Failures with Expected Failures | |
| run: | | |
| shopt -s globstar | |
| actual_failures=$(grep "FAILED tests" reports/**/*.txt | awk '{print $2}' | sort) | |
| expected_failures=$(sort .github/expected_failures.txt) | |
| echo "Expected Failures:" | |
| echo "$expected_failures" | |
| echo "" | |
| echo "Actual Failures:" | |
| echo "$actual_failures" | |
| echo "" | |
| # Identify unexpected failures | |
| unexpected_failures=$(comm -13 <(echo "$expected_failures") <(echo "$actual_failures")) | |
| # Identify missing expected failures | |
| missing_failures=$(comm -23 <(echo "$expected_failures") <(echo "$actual_failures")) | |
| # Initialize exit code | |
| exit_code=0 | |
| if [ -n "$unexpected_failures" ]; then | |
| echo "Unexpected test failures detected:" | |
| echo "$unexpected_failures" | |
| exit_code=1 | |
| fi | |
| if [ -n "$missing_failures" ]; then | |
| echo "Expected test failures that did not occur (they passed):" | |
| echo "$missing_failures" | |
| exit_code=1 | |
| fi | |
| if [ $exit_code -eq 0 ]; then | |
| echo "All failed tests are expected, and all expected failures have occurred." | |
| else | |
| echo "Verification failed: There are unexpected or missing test failures." | |
| fi | |
| exit $exit_code |