CDK Source Connector Compatibility Test #37
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CDK Source Connector Compatibility Test | |
| on: | |
| pull_request: | |
| paths: | |
| - "airbyte-cdk/bulk/core/base/version.properties" | |
| - "airbyte-cdk/bulk/core/extract/version.properties" | |
| schedule: | |
| - cron: "0 12 * * *" | |
| workflow_dispatch: | |
| jobs: | |
| generate-matrix: | |
| name: Generate Source Connector Matrix | |
| runs-on: ubuntu-24.04 | |
| outputs: | |
| connectors-matrix: ${{ steps.generate-matrix.outputs.connectors_matrix }} | |
| steps: | |
| - name: Checkout Airbyte | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| submodules: true # Needed for when airbyte-enterprise calls this workflow | |
| - name: Install yq | |
| run: sudo snap install yq | |
| - name: Generate Source Connector Matrix | |
| id: generate-matrix | |
| run: echo "connectors_matrix=$(tools/bin/bulk-cdk/get-certified-connectors.sh sources)" | tee -a $GITHUB_OUTPUT | |
| connectors-test: | |
| name: Test ${{ matrix.connector }} | |
| needs: [generate-matrix] | |
| if: needs.generate-matrix.outputs.connectors-matrix != '[]' | |
| runs-on: linux-24.04-large | |
| env: | |
| GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} | |
| GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID || vars.GCP_PROJECT_ID }} | |
| DD_TAGS: "connector.name:${{ matrix.connector }}" | |
| strategy: | |
| matrix: | |
| connector: ${{ fromJson(needs.generate-matrix.outputs.connectors-matrix) }} | |
| max-parallel: 5 | |
| fail-fast: false | |
| steps: | |
| - name: Checkout Airbyte | |
| if: matrix.connector | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| submodules: true # Needed for airbyte-enterprise connectors (no-op otherwise) | |
| - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 | |
| if: matrix.connector | |
| with: | |
| distribution: zulu | |
| java-version: 21 | |
| cache: gradle | |
| - uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5.0.2 | |
| if: matrix.connector | |
| with: | |
| gradle-version: "8.14" | |
| - name: Install the latest version of uv | |
| if: matrix.connector | |
| uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8.0.0 | |
| - name: Install Poe | |
| if: matrix.connector | |
| run: uv tool install poethepoet | |
| - name: Install connector dependencies | |
| if: matrix.connector | |
| working-directory: airbyte-integrations/connectors/${{ matrix.connector }} | |
| run: poe install | |
| - name: Install Ops CLI | |
| if: matrix.connector && (vars.GCP_PROJECT_ID != '') | |
| run: uv tool install --upgrade airbyte-internal-ops | |
| - name: Fetch connector secrets | |
| if: matrix.connector && (vars.GCP_PROJECT_ID != '') | |
| run: airbyte-ops secrets fetch ${{ matrix.connector }} | |
| - name: Run CDK upgrade | |
| if: matrix.connector | |
| run: ./gradlew :airbyte-integrations:connectors:${{ matrix.connector }}:upgradeCdk --cdkVersion=local | |
| - name: Run Unit Tests | |
| if: matrix.connector | |
| working-directory: airbyte-integrations/connectors/${{ matrix.connector }} | |
| run: poe test-unit-tests | |
| - name: Run Integration Tests | |
| if: matrix.connector | |
| working-directory: airbyte-integrations/connectors/${{ matrix.connector }} | |
| run: poe test-integration-tests |