chore: add Volta configuration to docusaurus (do not merge) #16800
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Connector CI | |
| on: | |
| pull_request: | |
| types: | |
| - opened | |
| - synchronize | |
| - reopened | |
| - ready_for_review | |
| # Available as a reusable workflow | |
| # (https://docs.github.com/en/actions/sharing-automations/reusing-workflows) | |
| workflow_call: | |
| inputs: | |
| repo: | |
| type: string | |
| required: false | |
| description: "The repository name" | |
| gitref: | |
| type: string | |
| required: false | |
| description: "The git reference (branch or tag)" | |
| comment-id: | |
| type: string | |
| required: false | |
| description: "The ID of the comment triggering the workflow. Unused as of now." | |
| pr: | |
| type: string | |
| required: false | |
| description: "The pull request number, if applicable. Unused as of now." | |
| outputs: | |
| result: | |
| description: "The result of the checks, e.g. 'success' if successful." | |
| value: ${{ jobs.connector-ci-checks-summary.result }} | |
| commit-sha: | |
| description: "The commit SHA of the current branch, used to report test results." | |
| value: ${{ jobs.generate-matrix.outputs.commit-sha }} | |
| permissions: | |
| # Allow the workflow to read contents | |
| # and to read/write checks and PR comments. | |
| contents: read | |
| checks: write | |
| pull-requests: write | |
| jobs: | |
| generate-matrix: | |
| name: Generate Connector Matrix | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: Checkout Current Branch | |
| id: checkout | |
| uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 | |
| with: | |
| repository: ${{ inputs.repo || github.event.pull_request.head.repo.full_name }} | |
| ref: ${{ inputs.gitref || github.head_ref || github.ref_name }} | |
| submodules: true # Needed for airbyte-enterprise connectors (no-op otherwise) | |
| # Use fetch-depth: 0 to ensure we can access the full commit history | |
| fetch-depth: 0 | |
| - name: Add upstream remote (forks only) | |
| # This step only runs from forks. | |
| # It ensures the `upstream` ref is properly declared before proceeding | |
| # to the step of determining which connectors are modified. | |
| if: > | |
| (inputs.repo != '' && | |
| ! startswith(inputs.repo, 'airbytehq/') | |
| ) || | |
| (github.event.pull_request.head.repo.full_name != '' && | |
| ! startswith(github.event.pull_request.head.repo.full_name, 'airbytehq/') | |
| ) | |
| run: | | |
| # only add upstream if it doesn't already exist | |
| if ! git remote | grep -q upstream; then | |
| git remote add upstream https://github.com/airbytehq/airbyte.git | |
| fi | |
| # fetch the default branch from upstream | |
| git fetch --quiet upstream master | |
| - id: cdk-changes | |
| uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1 | |
| with: | |
| # Note: expressions within a filter are OR'ed | |
| filters: | | |
| java-cdk: | |
| - 'airbyte-cdk/java/**/**/src/main/**/*' | |
| - 'airbyte-cdk/bulk/**/**/src/main/**/*' | |
| - name: Generate Connector Matrix from Changes | |
| id: generate-matrix | |
| run: | | |
| # Get the list of modified connectors | |
| echo "connectors_matrix=$(./poe-tasks/get-modified-connectors.sh --json)" | tee -a $GITHUB_OUTPUT | |
| # If local-cdk-changed is true, get Java connectors with useLocalCdk = true | |
| # Otherwise, get modified Java connectors | |
| if [[ "${{ steps.cdk-changes.outputs.java-cdk == 'true' }}" == "true" ]]; then | |
| echo "jvm_connectors_matrix=$(./poe-tasks/get-modified-connectors.sh --json --local-cdk)" | tee -a $GITHUB_OUTPUT | |
| else | |
| echo "jvm_connectors_matrix=$(./poe-tasks/get-modified-connectors.sh --json --java)" | tee -a $GITHUB_OUTPUT | |
| fi | |
| echo "non_jvm_connectors_matrix=$(./poe-tasks/get-modified-connectors.sh --json --no-java)" | tee -a $GITHUB_OUTPUT | |
| - name: Set JVM Runner Type | |
| id: set-jvm-runner-type | |
| run: | | |
| # Default to ubuntu-24.04 for all connectors | |
| echo "jvm-runner-type=ubuntu-24.04" >> $GITHUB_OUTPUT | |
| if [[ "${{ github.repository }}" == "airbytehq/airbyte" || "${{ github.repository }}" == "airbytehq/airbyte-enterprise" ]]; then | |
| # Within our own repo, we can use the custom (larger) runner | |
| echo "jvm-runner-type=linux-24.04-large" | tee -a "$GITHUB_OUTPUT" | |
| fi | |
| outputs: | |
| connectors-matrix: ${{ steps.generate-matrix.outputs.connectors_matrix }} | |
| jvm-connectors-matrix: ${{ steps.generate-matrix.outputs.jvm_connectors_matrix }} | |
| non-jvm-connectors-matrix: ${{ steps.generate-matrix.outputs.non_jvm_connectors_matrix }} | |
| jvm-runner-type: ${{ steps.set-jvm-runner-type.outputs.jvm-runner-type }} | |
| # The commit SHA of the current branch, used to report test results: | |
| commit-sha: ${{ steps.checkout.outputs.commit }} | |
| creds-available: ${{ (secrets.GCP_PROJECT_ID == '') && (vars.GCP_PROJECT_ID == '') && 'false' || 'true' }} | |
| jvm-connectors-test: | |
| needs: [generate-matrix] | |
| runs-on: ${{ needs.generate-matrix.outputs.jvm-runner-type }} # Custom runner, defined in GitHub org settings | |
| env: | |
| GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} | |
| GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID || vars.GCP_PROJECT_ID }} | |
| DD_TAGS: "connector.name:${{ matrix.connector }}" | |
| strategy: | |
| matrix: ${{ fromJson(needs.generate-matrix.outputs.jvm-connectors-matrix) }} | |
| max-parallel: 5 # Limit number of parallel jobs | |
| fail-fast: false # Don't stop on first failure | |
| name: Test ${{ matrix.connector }} Connector ${{ ! matrix.connector && ' (no JVM-based connectors modified)' || ( needs.generate-matrix.outputs.creds-available == 'false' && ' [No Creds]' || '') }} | |
| steps: | |
| - name: Checkout Airbyte | |
| if: matrix.connector | |
| id: checkout | |
| uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 | |
| with: | |
| repository: ${{ inputs.repo || github.event.pull_request.head.repo.full_name }} | |
| ref: ${{ inputs.gitref || github.head_ref || github.ref_name }} | |
| submodules: true # Needed for airbyte-enterprise connectors (no-op otherwise) | |
| fetch-depth: 1 | |
| # Java deps | |
| - uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1 | |
| if: matrix.connector | |
| with: | |
| distribution: zulu | |
| java-version: 21 | |
| cache: gradle | |
| # The default behaviour is read-only on PR branches and read/write on master. | |
| # See https://github.com/gradle/actions/blob/main/docs/setup-gradle.md#using-the-cache-read-only. | |
| - uses: gradle/actions/setup-gradle@748248ddd2a24f49513d8f472f81c3a07d4d50e1 # v4.4.4 | |
| if: matrix.connector | |
| with: | |
| gradle-version: "8.14" | |
| # TODO: We can delete this step once Airbyte-CI is removed from Java integration tests. | |
| - name: Set up Python (For Airbyte-CI) | |
| if: matrix.connector | |
| uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 | |
| with: | |
| python-version: "3.11" | |
| check-latest: true | |
| update-environment: true | |
| - name: Install the latest version of uv | |
| if: matrix.connector | |
| uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0 | |
| - name: Install Poe | |
| if: matrix.connector | |
| run: | | |
| # Install Poe so we can run the connector tasks: | |
| uv tool install poethepoet | |
| - name: Install connector dependencies | |
| if: matrix.connector | |
| working-directory: airbyte-integrations/connectors/${{ matrix.connector }} | |
| run: poe install | |
| - name: Fetch connector secrets | |
| # This will be skipped if the repo or fork does not set GCP_PROJECT_ID, or if it is set to an empty value. | |
| # Later integration tests will likely fail in this case, but we at least let them run. | |
| if: matrix.connector && needs.generate-matrix.outputs.creds-available == 'true' | |
| run: | | |
| airbyte-cdk secrets fetch ${{ matrix.connector }} \ | |
| --print-ci-secrets-masks | |
| - name: Run Unit Tests ${{ needs.generate-matrix.outputs.creds-available == 'false' && '[Non-Blocking (Unprivileged) Test From Fork, pls run `/run-connector-tests`]' || '' }} | |
| if: matrix.connector | |
| working-directory: airbyte-integrations/connectors/${{ matrix.connector }} | |
| run: poe test-unit-tests | |
| - name: Run Integration Tests ${{ needs.generate-matrix.outputs.creds-available == 'false' && '[Non-Blocking (Unprivileged) Test From Fork, pls run `/run-connector-tests`]' || '' }} | |
| if: matrix.connector | |
| working-directory: airbyte-integrations/connectors/${{ matrix.connector }} | |
| run: poe test-integration-tests | |
| - name: Publish Test Results | |
| uses: EnricoMi/publish-unit-test-result-action@3a74b2957438d0b6e2e61d67b05318aa25c9e6c6 # v2.20.0 | |
| # Skip if cancelled, if no connector is specified, or if running from a fork. | |
| if: always() && !cancelled() && matrix.connector && github.repository == 'airbytehq/airbyte' | |
| with: | |
| check_name: "`${{ matrix.connector }}` Connector Test Results" | |
| commit: ${{ steps.checkout.outputs.commit }} | |
| large_files: true | |
| files: | | |
| airbyte-integrations/connectors/${{ matrix.connector }}/build/test-results/**/*.xml | |
| non-jvm-connectors-test: | |
| needs: [generate-matrix] | |
| runs-on: ubuntu-24.04 | |
| env: | |
| GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} | |
| GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID || vars.GCP_PROJECT_ID }} | |
| strategy: | |
| matrix: ${{ fromJson(needs.generate-matrix.outputs.non-jvm-connectors-matrix) }} | |
| max-parallel: 5 # Limit number of parallel jobs | |
| fail-fast: false # Don't stop on first failure | |
| name: Test ${{ matrix.connector }} Connector ${{ ! matrix.connector && ' (no non-JVM-based connectors modified)' || ( needs.generate-matrix.outputs.creds-available == 'false' && ' [No Creds]' || '') }} | |
| steps: | |
| - name: Checkout Airbyte | |
| if: matrix.connector | |
| id: checkout | |
| uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 | |
| with: | |
| repository: ${{ inputs.repo || github.event.pull_request.head.repo.full_name }} | |
| ref: ${{ inputs.gitref || github.head_ref || github.ref_name }} | |
| submodules: true # Needed for airbyte-enterprise connectors (no-op otherwise) | |
| fetch-depth: 1 | |
| # Python deps | |
| - name: Set up Python | |
| if: matrix.connector | |
| uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 | |
| with: | |
| python-version: "3.11" | |
| check-latest: true | |
| update-environment: true | |
| - name: Install and configure Poetry | |
| if: matrix.connector | |
| uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1.4.1 | |
| with: | |
| version: 1.8.5 | |
| - name: Install the latest version of uv | |
| if: matrix.connector | |
| uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0 | |
| - name: Install system dependencies | |
| if: matrix.connector | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y \ | |
| build-essential \ | |
| libssl-dev \ | |
| libffi-dev \ | |
| python3-dev | |
| - name: Install Poe and Dev Dependencies | |
| if: matrix.connector | |
| run: | | |
| # Install Poe so we can run the connector tasks: | |
| uv tool install poethepoet | |
| # These should be included as dev dependencies in the connector, but | |
| # for many connectors they are undeclared: | |
| uv tool install ruff | |
| uv tool install pytest | |
| uv tool install mypy | |
| - name: Install connector dependencies | |
| if: matrix.connector | |
| working-directory: airbyte-integrations/connectors/${{ matrix.connector }} | |
| run: poe install | |
| - name: Fetch connector secrets | |
| # This will be skipped if the repo or fork does not set GCP_PROJECT_ID, or if it is set to an empty value. | |
| # Later integration tests will likely fail in this case, but we at least let them run. | |
| if: matrix.connector && env.GCP_PROJECT_ID != '' | |
| run: | | |
| airbyte-cdk secrets fetch ${{ matrix.connector }} \ | |
| --print-ci-secrets-masks | |
| - name: Run Unit Tests ${{ needs.generate-matrix.outputs.creds-available == 'false' && '[Non-Blocking (Unprivileged) Test From Fork, pls run `/run-connector-tests`]' || '' }} | |
| if: matrix.connector | |
| working-directory: airbyte-integrations/connectors/${{ matrix.connector }} | |
| run: poe test-unit-tests | |
| - name: Run Integration Tests ${{ needs.generate-matrix.outputs.creds-available == 'false' && '[Non-Blocking (Unprivileged) Test From Fork, pls run `/run-connector-tests`]' || '' }} | |
| if: matrix.connector | |
| working-directory: airbyte-integrations/connectors/${{ matrix.connector }} | |
| run: poe test-integration-tests | |
| - name: Container Tests | |
| if: matrix.connector | |
| working-directory: airbyte-integrations/connectors/${{ matrix.connector }} | |
| run: | | |
| airbyte-cdk image test | |
| - name: Publish Test Results | |
| uses: EnricoMi/publish-unit-test-result-action@3a74b2957438d0b6e2e61d67b05318aa25c9e6c6 # v2.20.0 | |
| # Skip if cancelled, if no connector is specified, or if the repository is a fork. | |
| if: always() && !cancelled() && matrix.connector && github.repository == 'airbytehq/airbyte' | |
| with: | |
| check_name: "`${{ matrix.connector }}` Connector Test Results" | |
| commit: ${{ steps.checkout.outputs.commit }} | |
| large_files: true | |
| files: | | |
| airbyte-integrations/connectors/${{ matrix.connector }}/build/test-results/**/*.xml | |
| # Lint and format checks | |
| connectors-lint: | |
| needs: [generate-matrix] | |
| runs-on: ubuntu-24.04 | |
| if: ${{ needs.generate-matrix.outputs.connectors-matrix }} | |
| strategy: | |
| matrix: ${{ fromJson(needs.generate-matrix.outputs.connectors-matrix) }} | |
| max-parallel: 10 # Limit number of parallel jobs | |
| name: Lint ${{ matrix.connector }} Connector ${{ ! matrix.connector && ' (no connectors modified)' || '' }} | |
| steps: | |
| - name: Checkout Airbyte | |
| if: matrix.connector | |
| uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 | |
| with: | |
| repository: ${{ inputs.repo || github.event.pull_request.head.repo.full_name }} | |
| ref: ${{ inputs.gitref || github.head_ref || github.ref_name }} | |
| submodules: true # Needed for airbyte-enterprise connectors (no-op otherwise) | |
| fetch-depth: 1 | |
| # Java deps | |
| - uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1 | |
| if: matrix.connector | |
| with: | |
| distribution: zulu | |
| java-version: 21 | |
| cache: gradle | |
| - uses: gradle/actions/setup-gradle@748248ddd2a24f49513d8f472f81c3a07d4d50e1 # v4.4.4 | |
| if: matrix.connector | |
| with: | |
| cache-read-only: false | |
| cache-write-only: false | |
| gradle-version: "8.14" | |
| # Python deps | |
| - name: Set up Python | |
| if: matrix.connector | |
| uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 | |
| with: | |
| python-version: "3.11" | |
| check-latest: true | |
| update-environment: true | |
| - name: Install and configure Poetry | |
| if: matrix.connector | |
| uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1.4.1 | |
| with: | |
| version: 1.8.5 | |
| - name: Install the latest version of uv | |
| if: matrix.connector | |
| uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0 | |
| - name: Install system dependencies | |
| if: matrix.connector | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y \ | |
| build-essential \ | |
| libssl-dev \ | |
| libffi-dev \ | |
| python3-dev | |
| - name: Install Poe and Dev Dependencies | |
| if: matrix.connector | |
| run: | | |
| # Install Poe so we can run the connector tasks: | |
| uv tool install poethepoet | |
| # These should be included as dev dependencies in the connector, but | |
| # for many connectors they are undeclared: | |
| uv tool install ruff | |
| uv tool install pytest | |
| uv tool install mypy | |
| - name: Install connector dependencies | |
| if: matrix.connector | |
| working-directory: airbyte-integrations/connectors/${{ matrix.connector }} | |
| run: poe install | |
| - name: Run format-check | |
| if: matrix.connector | |
| working-directory: airbyte-integrations/connectors/${{ matrix.connector }} | |
| run: poe format-check | |
| - name: Run lint check (info only) | |
| if: matrix.connector | |
| working-directory: airbyte-integrations/connectors/${{ matrix.connector }} | |
| run: poe lint-check | |
| # Most connectors can't pass lint checks, so this is non-blocking for now | |
| continue-on-error: true | |
| pre-release-checks: | |
| if: github.event.pull_request.draft == false | |
| needs: [generate-matrix] | |
| strategy: | |
| matrix: ${{ fromJson(needs.generate-matrix.outputs.connectors-matrix) }} | |
| max-parallel: 10 # Limit number of parallel jobs | |
| fail-fast: false # Don't stop on first failure | |
| name: ${{ matrix.connector || 'No-Op' }} Pre-Release Checks | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: Checkout Airbyte | |
| if: matrix.connector | |
| uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 | |
| with: | |
| repository: ${{ inputs.repo || github.event.pull_request.head.repo.full_name }} | |
| ref: ${{ inputs.gitref || github.head_ref || github.ref_name }} | |
| submodules: true # Needed for airbyte-enterprise connectors (no-op otherwise) | |
| fetch-depth: 0 | |
| - name: Install uv | |
| if: matrix.connector | |
| uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0 | |
| - name: Install Poe | |
| if: matrix.connector | |
| run: | | |
| # Install Poe so we can run the connector tasks: | |
| uv tool install poethepoet | |
| - name: Install QA Checks | |
| if: matrix.connector | |
| run: | | |
| uv tool install airbyte-ci/connectors/connectors_qa | |
| - name: Run QA Checks | |
| if: matrix.connector | |
| # remove "-strict-encrypt" suffix, if present, for parity with the previous version of this workflow | |
| run: | | |
| connector_name=${{ matrix.connector }} | |
| connectors-qa run --name ${connector_name%-strict-encrypt} | |
| - name: Detect Python CDK Prerelease Versions | |
| if: matrix.connector | |
| working-directory: airbyte-integrations/connectors/${{ matrix.connector }} | |
| run: | | |
| # Exit with code 1 if the CDK is not pinned to a standard version. | |
| # This is a no-op for non-Python connectors. | |
| if [[ $(poe -qq get-language) == "python" ]]; then | |
| poe detect-cdk-prerelease | |
| else | |
| echo "Skipping CDK pre-release check for non-Python connector." | |
| fi | |
| connector-ci-checks-summary: | |
| name: Aggregate Results | |
| if: always() | |
| needs: | |
| - generate-matrix | |
| - jvm-connectors-test | |
| - non-jvm-connectors-test | |
| - pre-release-checks | |
| - connectors-lint | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: Evaluate Status | |
| id: evaluate-status | |
| run: | | |
| if [[ "${{ needs.jvm-connectors-test.result }}" == "success" && | |
| "${{ needs.non-jvm-connectors-test.result }}" == "success" && | |
| "${{ needs.connectors-lint.result }}" == "success" ]]; then | |
| echo "result=success" | tee -a $GITHUB_OUTPUT | |
| else | |
| echo "result=failure" | tee -a $GITHUB_OUTPUT | |
| fi | |
| # In slash commands and when not running within a fork, we want to upload | |
| # a check status report to the PR. | |
| # If we are not running from the airbytehq/airbyte repo, the secrets won't | |
| # be available, so the following steps will be a no-op. | |
| - name: Authenticate as GitHub App | |
| uses: actions/create-github-app-token@d72941d797fd3113feb6b93fd0dec494b13a2547 # v1.12.0 | |
| id: get-app-token | |
| if: always() && !cancelled() | |
| continue-on-error: true | |
| with: | |
| app-id: ${{ secrets.OCTAVIA_BOT_APP_ID }} | |
| private-key: ${{ secrets.OCTAVIA_BOT_PRIVATE_KEY }} | |
| - name: Create Check Status | |
| if: > | |
| always() && !cancelled() && | |
| steps.get-app-token.outcome == 'success' | |
| uses: LouisBrunner/checks-action@6b626ffbad7cc56fd58627f774b9067e6118af23 # v2.0.0 | |
| with: | |
| name: "Connector CI Checks Summary" # << Name of the 'Required' check | |
| sha: ${{ needs.generate-matrix.outputs.commit-sha }} | |
| status: completed | |
| conclusion: ${{ steps.evaluate-status.outputs.result }} | |
| token: ${{ steps.get-app-token.outputs.token }} | |
| details_url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} |