source-stripe-native: speed up fetching connected account ids with co… #12336
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| branches: [main] | |
| jobs: | |
| build_base_image: | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 0 | |
| - name: Prepare | |
| id: prep | |
| run: | | |
| TAG=$(echo $GITHUB_SHA | head -c7) | |
| echo ::set-output name=tag::${TAG} | |
| VERSION=$(cat base-image/VERSION | tr -d '\n') | |
| echo ::set-output name=version::${VERSION} | |
| - name: Login to GitHub package docker registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v1 | |
| with: | |
| driver-opts: | | |
| network=host | |
| - name: Build base-image Docker Image | |
| uses: docker/build-push-action@v2 | |
| with: | |
| context: . | |
| file: base-image/Dockerfile | |
| load: true | |
| tags: ghcr.io/estuary/base-image:local | |
| - name: Push base-image image | |
| uses: docker/build-push-action@v2 | |
| with: | |
| context: . | |
| file: base-image/Dockerfile | |
| push: true | |
| tags: ghcr.io/estuary/base-image:${{ steps.prep.outputs.tag }} | |
| - name: Push base-image image with 'dev' tag | |
| if: ${{ github.event_name == 'push' }} | |
| uses: docker/build-push-action@v2 | |
| with: | |
| context: . | |
| file: base-image/Dockerfile | |
| push: true # See 'if' above | |
| tags: ghcr.io/estuary/base-image:dev,ghcr.io/estuary/base-image:${{ steps.prep.outputs.version }} | |
| build_connectors: | |
| runs-on: ubuntu-24.04 | |
| needs: build_base_image | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| connector: | |
| - source-alpaca | |
| - source-bigquery-batch | |
| - source-db2-batch | |
| - source-dropbox | |
| - source-dynamodb | |
| - source-firestore | |
| - source-gcs | |
| - source-google-drive | |
| - source-google-pubsub | |
| - source-hello-world | |
| - source-http-file | |
| - source-http-ingest | |
| - source-kafka | |
| - source-kinesis | |
| - source-mongodb | |
| - source-mysql | |
| - source-mysql-batch | |
| - source-onedrive | |
| - source-oracle | |
| - source-oracle-batch | |
| - source-postgres | |
| - source-postgres-batch | |
| - source-redshift-batch | |
| - source-s3 | |
| - source-sftp | |
| - source-sharepoint | |
| - source-snowflake | |
| - source-sqlserver | |
| - source-sqlserver-batch | |
| - source-sqlserver-ct | |
| - source-test | |
| - source-azure-blob-storage | |
| - materialize-azure-blob-parquet | |
| - materialize-azure-fabric-warehouse | |
| - materialize-bigquery | |
| - materialize-clickhouse | |
| - materialize-cratedb | |
| - materialize-databricks | |
| - materialize-dynamodb | |
| - materialize-elasticsearch | |
| - materialize-firebolt | |
| - materialize-gcs-csv | |
| - materialize-gcs-parquet | |
| - materialize-google-pubsub | |
| - materialize-google-sheets | |
| - materialize-iceberg | |
| - materialize-kafka | |
| - materialize-mongodb | |
| - materialize-motherduck | |
| - materialize-mysql | |
| - materialize-pinecone | |
| - materialize-postgres | |
| - materialize-redshift | |
| - materialize-s3-csv | |
| - materialize-s3-iceberg | |
| - materialize-s3-parquet | |
| - materialize-slack | |
| - materialize-snowflake | |
| - materialize-spanner | |
| - materialize-sqlite | |
| - materialize-sqlserver | |
| - materialize-starburst | |
| - materialize-webhook | |
| include: | |
| - connector: source-criteo | |
| connector_type: capture | |
| python: true | |
| - connector: source-shopify | |
| connector_type: capture | |
| python: true | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 0 | |
| - name: Prepare | |
| id: prep | |
| run: | | |
| TAG=$(echo $GITHUB_SHA | head -c7) | |
| echo ::set-output name=tag::${TAG} | |
| VERSION=$(cat ${{ matrix.connector }}/VERSION | tr -d '\n') | |
| echo ::set-output name=version::${VERSION} | |
| VARIANTS="${{ matrix.connector }} $(cat ${{ matrix.connector }}/VARIANTS 2>/dev/null || true)" | |
| echo ::set-output name=variants::${VARIANTS} | |
| - name: Download latest Flow release binaries and add them to $PATH | |
| run: | | |
| ./fetch-flow.sh | |
| echo "${PWD}/flow-bin" >> $GITHUB_PATH | |
| - name: Set up Cloud SDK | |
| uses: google-github-actions/setup-gcloud@v0 | |
| with: | |
| project_id: ${{ secrets.GCP_PROJECT_ID }} | |
| service_account_key: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }} | |
| export_default_credentials: true | |
| - name: Login to GitHub package docker registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v1 | |
| with: | |
| driver-opts: | | |
| network=host | |
| - name: Create docker network flow-test | |
| run: docker network create flow-test | |
| - name: Start Dockerized test infrastructure | |
| if: | | |
| contains(fromJson('[ | |
| "materialize-clickhouse", | |
| "materialize-dynamodb", | |
| "materialize-elasticsearch", | |
| "materialize-kafka", | |
| "source-db2-batch", | |
| "source-dynamodb", | |
| "source-kafka", | |
| "source-kinesis", | |
| "source-mysql", | |
| "source-mysql-batch", | |
| "source-postgres", | |
| "source-postgres-batch", | |
| "source-sftp", | |
| "source-sqlserver", | |
| "source-sqlserver-batch", | |
| "source-sqlserver-ct", | |
| "source-mongodb" | |
| ]'), matrix.connector) | |
| run: | | |
| set +e # Show logs even if startup fails | |
| docker compose --file ${{ matrix.connector }}/docker-compose.yaml up --wait | |
| docker logs ${{ matrix.connector }}-db-1 | |
| - name: Test setup | |
| if: matrix.connector == 'source-mongodb' | |
| run: | | |
| bash source-mongodb/docker-compose-init.sh | |
| # Go connector tests which aren't run as part of Dockerfile builds. Most Go connectors | |
| # run their tests as part of the Docker build, but running the tests outside of there | |
| # makes GCP credentials plumbing for encrypted configs significantly simpler. | |
| - name: Go Connector ${{ matrix.connector }} Database Tests | |
| if: | | |
| contains(fromJson('[ | |
| "source-postgres", | |
| "source-redshift-batch", | |
| "source-sqlserver", | |
| "source-sqlserver-ct" | |
| ]'), matrix.connector) | |
| run: | | |
| ${{ matrix.connector }}/run_tests.sh | |
| env: | |
| GCP_SERVICE_ACCOUNT_KEY: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }} | |
| # The estuary/flow repository contains a few large files in Git LFS. This can | |
| # present an issue with Go module checksums when using a commit-tagged version, | |
| # because a commit-tagged version will be fetched from VCS using `git archive` | |
| # and the resulting data will differ depending on whether Git LFS is installed | |
| # on the machine running the build, and so there are two "valid" checksums but | |
| # only one can match what's in go.sum / sum.golang.org | |
| # | |
| # When we're using a proper release version this isn't an issue, because there's | |
| # a prebuilt zip file available from proxy.golang.org which will be the same for | |
| # everyone. But we set this here so that the behavior in the GitHub CI runner | |
| # (which has LFS installed) will match the Docker builder we use (which doesn't) | |
| # even when using a commit pseudo-version. | |
| GIT_LFS_SKIP_SMUDGE: 1 | |
| - name: Build ${{ matrix.connector }} Docker Image | |
| uses: docker/build-push-action@v2 | |
| if: ${{!matrix.python}} | |
| with: | |
| context: . | |
| file: ${{ matrix.connector }}/Dockerfile | |
| load: true | |
| build-args: BASE_IMAGE=ghcr.io/estuary/base-image:${{ steps.prep.outputs.tag }} | |
| tags: ghcr.io/estuary/${{ matrix.connector }}:local | |
| - name: Build ${{ matrix.connector }} Python Docker Image | |
| uses: docker/build-push-action@v2 | |
| if: ${{matrix.python}} | |
| with: | |
| context: . | |
| file: python/Dockerfile | |
| load: true | |
| build-args: | | |
| CONNECTOR_NAME=${{matrix.connector}} | |
| CONNECTOR_TYPE=${{matrix.connector_type}} | |
| tags: ghcr.io/estuary/${{ matrix.connector }}:local | |
| - name: Setup Python for ${{ matrix.connector }} | |
| uses: actions/setup-python@v5 | |
| if: ${{matrix.python}} | |
| with: | |
| python-version: '3.11' | |
| - name: Install Poetry | |
| if: ${{matrix.python}} | |
| uses: snok/install-poetry@v1 | |
| - name: Python connector ${{ matrix.connector }} snapshot tests | |
| if: ${{matrix.python}} | |
| run: | | |
| cd ${{matrix.connector}}; | |
| poetry install; | |
| source $(poetry env info --path)/bin/activate | |
| cd ..; | |
| pytest ${{matrix.connector}}/tests; | |
| - name: Source connector ${{ matrix.connector }} integration tests | |
| if: | | |
| contains(fromJson('[ | |
| "source-dynamodb", | |
| "source-gcs", | |
| "source-kinesis", | |
| "source-mysql", | |
| "source-postgres", | |
| "source-kafka", | |
| "source-s3", | |
| "source-sftp", | |
| "source-sqlserver" | |
| ]'), matrix.connector) | |
| env: | |
| GCP_SERVICE_ACCOUNT_KEY: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }} | |
| MYSQL_DATABASE: test | |
| run: CONNECTOR=${{ matrix.connector }} VERSION=local ./tests/run.sh; | |
| - name: Install duckdb | |
| if: matrix.connector == 'materialize-motherduck' || matrix.connector == 'materialize-s3-iceberg' | |
| uses: opt-nc/setup-duckdb-action@v1.0.8 | |
| with: | |
| version: v1.3.1 | |
| - name: Install iceberg-ctl Python dependencies | |
| if: matrix.connector == 'materialize-s3-iceberg' | |
| run: | | |
| pip install --break-system-packages poetry | |
| cd materialize-s3-iceberg/iceberg-ctl && poetry config virtualenvs.in-project true && poetry install | |
| - name: Install system dependencies for Rust connectors | |
| if: matrix.connector == 'materialize-kafka' | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y pkg-config cmake libssl-dev libsasl2-dev libclang-dev | |
| - name: Materialization connector ${{ matrix.connector }} integration tests | |
| if: | | |
| contains(fromJson('[ | |
| "materialize-azure-fabric-warehouse", | |
| "materialize-bigquery", | |
| "materialize-clickhouse", | |
| "materialize-databricks", | |
| "materialize-dynamodb", | |
| "materialize-elasticsearch", | |
| "materialize-firebolt", | |
| "materialize-google-sheets", | |
| "materialize-iceberg", | |
| "materialize-kafka", | |
| "materialize-mongodb", | |
| "materialize-motherduck", | |
| "materialize-mysql", | |
| "materialize-pinecone", | |
| "materialize-postgres", | |
| "materialize-redshift", | |
| "materialize-s3-iceberg", | |
| "materialize-s3-parquet", | |
| "materialize-snowflake", | |
| "materialize-spanner", | |
| "materialize-sqlserver" | |
| ]'), matrix.connector) | |
| run: | | |
| if [ "${{ matrix.connector }}" = "materialize-s3-iceberg" ]; then | |
| export PYTHON_PATH=$(pwd)/materialize-s3-iceberg/iceberg-ctl/.venv/bin/python | |
| fi | |
| CONNECTOR=${{ matrix.connector }} VERSION=local tests/materialize/run.sh; | |
| - name: Push ${{ matrix.connector }} image(s) with commit SHA tag | |
| run: | | |
| for VARIANT in ${{ steps.prep.outputs.variants }}; do | |
| echo "Building and pushing ${VARIANT}:${{ steps.prep.outputs.tag }}..."; | |
| docker build --build-arg BASE_CONNECTOR=ghcr.io/estuary/${{ matrix.connector }}:local \ | |
| --build-arg DOCS_URL=https://go.estuary.dev/${VARIANT} \ | |
| --tag ghcr.io/estuary/${VARIANT}:${{ steps.prep.outputs.tag }} \ | |
| --file connector-variant.Dockerfile .; | |
| docker image push ghcr.io/estuary/${VARIANT}:${{ steps.prep.outputs.tag }}; | |
| done | |
| - name: Push ${{ matrix.connector }} image(s) with 'dev' and '${{ steps.prep.outputs.version }}' tags | |
| if: ${{ github.event_name == 'push' }} | |
| run: | | |
| for VARIANT in ${{ steps.prep.outputs.variants }}; do | |
| docker image tag ghcr.io/estuary/${VARIANT}:${{ steps.prep.outputs.tag }} ghcr.io/estuary/${VARIANT}:dev; | |
| docker image tag ghcr.io/estuary/${VARIANT}:${{ steps.prep.outputs.tag }} ghcr.io/estuary/${VARIANT}:${{ steps.prep.outputs.version }}; | |
| docker image push ghcr.io/estuary/${VARIANT}:dev; | |
| docker image push ghcr.io/estuary/${VARIANT}:${{ steps.prep.outputs.version }}; | |
| done | |
| - name: Install psql | |
| if: ${{ github.event_name == 'push' }} | |
| run: | | |
| sudo apt update | |
| sudo apt install postgresql | |
| - uses: dorny/paths-filter@v2 | |
| id: filter | |
| with: | |
| filters: | | |
| connector: | |
| - "${{ matrix.connector }}/**" | |
| - name: Refresh connector tags for ${{ matrix.connector }} | |
| if: github.event_name == 'push' && steps.filter.outputs.connector == 'true' | |
| env: | |
| PGHOST: ${{ secrets.POSTGRES_CONNECTOR_REFRESH_HOST }} | |
| PGUSER: ${{ secrets.POSTGRES_CONNECTOR_REFRESH_USER }} | |
| PGPASSWORD: ${{ secrets.POSTGRES_CONNECTOR_REFRESH_PASSWORD }} | |
| PGDATABASE: ${{ secrets.POSTGRES_CONNECTOR_REFRESH_DATABASE }} | |
| run: | | |
| for VARIANT in ${{ steps.prep.outputs.variants }}; do | |
| echo "UPDATE connector_tags SET job_status='{\"type\": \"queued\"}' | |
| WHERE connector_id IN ( | |
| SELECT id FROM connectors WHERE image_name='ghcr.io/estuary/${VARIANT}' | |
| ) AND image_tag IN (':${{ steps.prep.outputs.version }}', ':dev');" | psql; | |
| done |