dbt-dremio/.github/workflows/ci.yml at main · dremio/dbt-dremio · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
name: CI

on:
  push:
    branches:
      - main
  pull_request:
    branches:
      - main

jobs:
  discover-tests:
    name: Discover Tests
    runs-on: ubuntu-latest
    outputs:
      test_dirs: ${{ steps.discover-tests.outputs.test_dirs }}
    steps:
      - name: Check out repository
        uses: actions/checkout@v4

      - name: Discover Test Directories
        id: discover-tests
        run: |
          test_dirs=$(find tests/ -type f \( -name 'test_*.py' -o -name '*_test.py' \) -exec dirname {} \; | sort -u | jq -R -s -c 'split("\n") | map(select(length > 0))')
          echo "test_dirs=$test_dirs" >> $GITHUB_OUTPUT

  run-parallel-tests:
    name: Run Parallel Tests
    runs-on: ubuntu-latest
    continue-on-error: true
    needs: discover-tests
    env:
      RETRY_COUNT: 12 # number of retries for health checks
      SLEEP_INTERVAL: 5 # Sleep duration in seconds between retries
      MINIO_HOST_URL: http://localhost:9000
      MINIO_HEALTH_URL: http://localhost:9000/minio/health/live
      DREMIO_HEALTH_URL: http://localhost:9047
      MINIO_ROOT_USER: admin
      MINIO_ROOT_PASSWORD: password
      DREMIO_SOFTWARE_USERNAME: dremio
      DREMIO_SOFTWARE_PASSWORD: dremio123
      DREMIO_SOFTWARE_HOST: localhost
      DREMIO_DATALAKE: dbt_test_source
      DREMIO_DATABASE: dbt_test
      DBT_TEST_USER_1: dbt_test_user_1
      DBT_TEST_USER_2: dbt_test_user_2
      DBT_TEST_USER_3: dbt_test_user_3
    strategy:
      matrix:
        test_dir: ${{ fromJson(needs.discover-tests.outputs.test_dirs) }}
    services:
      minio:
        image: minio/minio:edge-cicd
        options: --network-alias minio
        ports:
          - 9000:9000
          - 9001:9001
        env:
          MINIO_ROOT_USER: ${{ env.MINIO_ROOT_USER }}
          MINIO_ROOT_PASSWORD: ${{ env.MINIO_ROOT_PASSWORD }}
      dremio:
        image: dremio/dremio-oss
        options: --network-alias dremio
        ports:
          - 31010:31010
          - 9047:9047
        env:
          DREMIO_JAVA_SERVER_EXTRA_OPTS: -Ddebug.addDefaultUser=true
    steps:
      - name: Check out repository
        uses: actions/checkout@v4

      - name: Install MinIO Client (mc)
        run: bash .github/scripts/install_minio_client.sh

      - name: Create MinIO bucket
        run: bash .github/scripts/create_minio_bucket.sh

      - name: Extract Auth Token
        run: bash .github/scripts/extract_auth_token.sh

      - name: Create and Format Sources
        run: bash .github/scripts/create_and_format_sources.sh

      - name: Install Dependencies
        uses: actions/setup-python@v4
        with:
          python-version: '3.9'

      - name: Install Python Dependencies
        run: |
          pip install --upgrade pip
          pip install -r dev_requirements.txt
          pip install .

      - name: Create dbt projects
        run: bash .github/scripts/create_dbt_projects.sh

      - name: Clean up __pycache__ directories
        run: |
          find . -type d -name "__pycache__" -exec rm -r {} +

      - name: Create .env file for tests
        run: bash .github/scripts/create_env_file.sh

      - name: Sanitize test directory name
        id: sanitize
        run: echo "sanitized_test_dir=$(echo ${{ matrix.test_dir }} | tr '/' '_')" >> $GITHUB_ENV

      - name: Run tests
        id: run_tests
        run: |
          mkdir reports/
          report_file="reports/${{ env.sanitized_test_dir }}.txt"
          pytest -s ${{ matrix.test_dir }} | tee $report_file
          exit ${PIPESTATUS[0]}

      - name: Upload test report as artifact
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ env.sanitized_test_dir }}
          path: reports/

  verify-failures:
    name: Verify Expected Test Failures
    runs-on: ubuntu-latest
    needs: run-parallel-tests
    steps:
      - name: Check out repository
        uses: actions/checkout@v3.5.2

      - name: Download All Test Reports
        uses: actions/download-artifact@v4
        with:
          path: reports/

      - name: Compare Actual Failures with Expected Failures
        run: bash .github/scripts/compare_test_failures.sh .github/expected_failures.txt