feat: add error index reporting for source hydration stage #28223
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Tests | |
| on: | |
| push: | |
| branches: | |
| - master | |
| - main | |
| - "release/*" | |
| pull_request: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.sha }} | |
| cancel-in-progress: true | |
| env: | |
| FB_TEST_PAGE_ACCESS_TOKEN: ${{ secrets.FB_TEST_PAGE_ACCESS_TOKEN }} | |
| permissions: | |
| contents: read # Required for actions/checkout | |
| jobs: | |
| integration: | |
| name: Integration | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| FEATURES: [ oss ,enterprise ] | |
| steps: | |
| - name: Harden the runner (Audit all outbound calls) | |
| uses: step-security/harden-runner@f4a75cfd619ee5ce8d5b864b0d183aff3c69b55a # v2.13.1 | |
| with: | |
| egress-policy: audit | |
| - name: Disable IPv6 (temporary fix) | |
| run: | | |
| sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1 | |
| sudo sysctl -w net.ipv6.conf.default.disable_ipv6=1 | |
| - name: Checkout | |
| uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 | |
| - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 | |
| with: | |
| go-version-file: 'go.mod' | |
| - run: go version | |
| - run: go mod download # Not required, used to segregate module download vs test times | |
| - name: enterprise | |
| if: matrix.FEATURES == 'enterprise' | |
| run: go test -v ./integration_test/docker_test/docker_test.go -count 1 | |
| env: | |
| ENTERPRISE_TOKEN: ${{ secrets.ENTERPRISE_TOKEN }} | |
| DOCKER_REGISTRY_MIRROR_USERNAME: ${{ vars.DOCKER_REGISTRY_MIRROR_USERNAME }} | |
| DOCKER_REGISTRY_MIRROR_PASSWORD: ${{ secrets.DOCKER_REGISTRY_MIRROR_PASSWORD }} | |
| DOCKER_REGISTRY_MIRROR: ${{ vars.DOCKER_REGISTRY_MIRROR }} | |
| - name: oss | |
| if: matrix.FEATURES == 'oss' | |
| run: go test -v ./integration_test/docker_test/docker_test.go -count 1 | |
| env: | |
| RSERVER_ENABLE_MULTITENANCY: false | |
| DOCKER_REGISTRY_MIRROR_USERNAME: ${{ vars.DOCKER_REGISTRY_MIRROR_USERNAME }} | |
| DOCKER_REGISTRY_MIRROR_PASSWORD: ${{ secrets.DOCKER_REGISTRY_MIRROR_PASSWORD }} | |
| DOCKER_REGISTRY_MIRROR: ${{ vars.DOCKER_REGISTRY_MIRROR }} | |
| warehouse-integration: | |
| name: Warehouse Integration | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| destination: [ bigquery, clickhouse, datalake, deltalake, mssql, azure-synapse, postgres, redshift, snowflake ] | |
| include: | |
| - package: warehouse/integrations/bigquery | |
| destination: bigquery | |
| - package: warehouse/integrations/clickhouse | |
| destination: clickhouse | |
| - package: warehouse/integrations/datalake | |
| destination: datalake | |
| - package: warehouse/integrations/deltalake | |
| destination: deltalake | |
| - package: warehouse/integrations/mssql | |
| destination: mssql | |
| - package: warehouse/integrations/azure-synapse | |
| destination: azure-synapse | |
| - package: warehouse/integrations/postgres | |
| destination: postgres | |
| - package: warehouse/integrations/redshift | |
| destination: redshift | |
| - package: warehouse/integrations/snowflake | |
| destination: snowflake | |
| steps: | |
| - name: Harden the runner (Audit all outbound calls) | |
| uses: step-security/harden-runner@f4a75cfd619ee5ce8d5b864b0d183aff3c69b55a # v2.13.1 | |
| with: | |
| egress-policy: audit | |
| - name: Disable IPv6 (temporary fix) | |
| run: | | |
| sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1 | |
| sudo sysctl -w net.ipv6.conf.default.disable_ipv6=1 | |
| - name: Checkout | |
| uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 | |
| - name: Setup Go | |
| uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 | |
| with: | |
| go-version-file: 'go.mod' | |
| - run: go version | |
| - run: go mod download # Not required, used to segregate module download vs test times | |
| - name: Login to DockerHub | |
| uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 | |
| with: | |
| username: ${{ vars.DOCKERHUB_READONLY_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_READONLY_TOKEN }} | |
| - name: Set Redshift Ginkgo AWS Credentials | |
| if: matrix.destination == 'redshift' | |
| run: | | |
| echo "Setting Redshift Ginkgo AWS Credentials..." | |
| echo "AWS_ACCESS_KEY_ID=${{ secrets.AWS_REDSHIFT_GINKGO_ACCESS_KEY_ID }}" >> $GITHUB_ENV | |
| echo "AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_REDSHIFT_GINKGO_SECRET_ACCESS_KEY }}" >> $GITHUB_ENV | |
| - name: Warehouse Service Integration [ ${{ matrix.destination }} ] | |
| run: make test-warehouse package=${{ matrix.package }} | |
| env: | |
| BIGQUERY_INTEGRATION_TEST_CREDENTIALS: ${{ secrets.BIGQUERY_INTEGRATION_TEST_CREDENTIALS }} | |
| DATABRICKS_INTEGRATION_TEST_CREDENTIALS: ${{ secrets.DATABRICKS_INTEGRATION_TEST_CREDENTIALS }} | |
| REDSHIFT_INTEGRATION_TEST_CREDENTIALS: ${{ secrets.REDSHIFT_INTEGRATION_TEST_CREDENTIALS }} | |
| REDSHIFT_IAM_INTEGRATION_TEST_CREDENTIALS: ${{ secrets.REDSHIFT_IAM_INTEGRATION_TEST_CREDENTIALS }} | |
| REDSHIFT_SERVERLESS_INTEGRATION_TEST_CREDENTIALS: ${{ secrets.REDSHIFT_SERVERLESS_INTEGRATION_TEST_CREDENTIALS }} | |
| REDSHIFT_SERVERLESS_IAM_INTEGRATION_TEST_CREDENTIALS: ${{ secrets.REDSHIFT_SERVERLESS_IAM_INTEGRATION_TEST_CREDENTIALS }} | |
| SNOWFLAKE_INTEGRATION_TEST_CREDENTIALS: ${{ secrets.SNOWFLAKE_INTEGRATION_TEST_CREDENTIALS }} | |
| SNOWFLAKE_RBAC_INTEGRATION_TEST_CREDENTIALS: ${{ secrets.SNOWFLAKE_RBAC_INTEGRATION_TEST_CREDENTIALS }} | |
| SNOWFLAKE_KEYPAIR_UNENCRYPTED_INTEGRATION_TEST_CREDENTIALS: ${{ secrets.SNOWFLAKE_KEYPAIR_UNENCRYPTED_INTEGRATION_TEST_CREDENTIALS }} | |
| RSERVER_FILE_MANAGER_USE_AWS_SDK_V2: "true" | |
| DOCKER_REGISTRY_MIRROR_USERNAME: ${{ vars.DOCKER_REGISTRY_MIRROR_USERNAME }} | |
| DOCKER_REGISTRY_MIRROR_PASSWORD: ${{ secrets.DOCKER_REGISTRY_MIRROR_PASSWORD }} | |
| DOCKER_REGISTRY_MIRROR: ${{ vars.DOCKER_REGISTRY_MIRROR }} | |
| RACE_ENABLED: "true" | |
| FORCE_RUN_INTEGRATION_TESTS: "true" | |
| - name: Upload coverage report | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: ${{ matrix.destination }} | |
| path: coverage.txt | |
| unit: | |
| name: Unit | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Harden the runner (Audit all outbound calls) | |
| uses: step-security/harden-runner@f4a75cfd619ee5ce8d5b864b0d183aff3c69b55a # v2.13.1 | |
| with: | |
| egress-policy: audit | |
| - name: Disable IPv6 (temporary fix) | |
| run: | | |
| sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1 | |
| sudo sysctl -w net.ipv6.conf.default.disable_ipv6=1 | |
| - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 | |
| - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 | |
| with: | |
| go-version-file: 'go.mod' | |
| - run: go version | |
| - run: go mod download # Not required, used to segregate module download vs test times | |
| - name: tests | |
| run: make test exclude="/rudder-server/(cluster|jobsdb|integration_test|processor|regulation-worker|router|services|suppression-backup-service|warehouse)" | |
| env: | |
| RSERVER_PROCESSOR_ENABLE_CONCURRENT_STORE: "true" | |
| DOCKER_REGISTRY_MIRROR_USERNAME: ${{ vars.DOCKER_REGISTRY_MIRROR_USERNAME }} | |
| DOCKER_REGISTRY_MIRROR_PASSWORD: ${{ secrets.DOCKER_REGISTRY_MIRROR_PASSWORD }} | |
| DOCKER_REGISTRY_MIRROR: ${{ vars.DOCKER_REGISTRY_MIRROR }} | |
| FORCE_RUN_INTEGRATION_TESTS: "true" | |
| - name: Upload coverage report | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: unit | |
| path: coverage.txt | |
| package-unit: | |
| name: Package Unit | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| package: | |
| - cluster | |
| - jobsdb | |
| - integration_test/docker_test | |
| - integration_test/multi_tenant_test | |
| - integration_test/reporting_dropped_events | |
| - integration_test/reporting_error_index | |
| - integration_test/warehouse | |
| - integration_test/tracing | |
| - integration_test/backendconfigunavailability | |
| - integration_test/trackedusersreporting | |
| - integration_test/snowpipestreaming | |
| - integration_test/srchydration | |
| - processor | |
| - regulation-worker | |
| - router | |
| - services | |
| - services/rsources | |
| - services/dedup | |
| - suppression-backup-service | |
| - warehouse | |
| include: | |
| - package: services | |
| exclude: services/rsources | |
| - package: services | |
| exclude: services/dedup | |
| - package: warehouse | |
| race: true | |
| steps: | |
| - name: Harden the runner (Audit all outbound calls) | |
| uses: step-security/harden-runner@f4a75cfd619ee5ce8d5b864b0d183aff3c69b55a # v2.13.1 | |
| with: | |
| egress-policy: audit | |
| - name: Disable IPv6 (temporary fix) | |
| if: matrix.package != 'services/dedup' | |
| run: | | |
| sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1 | |
| sudo sysctl -w net.ipv6.conf.default.disable_ipv6=1 | |
| - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 | |
| - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 | |
| with: | |
| go-version-file: 'go.mod' | |
| - run: go version | |
| - run: go mod download | |
| - name: Login to DockerHub | |
| uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 | |
| with: | |
| username: ${{ vars.DOCKERHUB_READONLY_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_READONLY_TOKEN }} | |
| - name: Package Unit [ ${{ matrix.package }} ] | |
| env: | |
| TEST_KAFKA_CONFLUENT_CLOUD_HOST: ${{ secrets.TEST_KAFKA_CONFLUENT_CLOUD_HOST }} | |
| TEST_KAFKA_CONFLUENT_CLOUD_KEY: ${{ secrets.TEST_KAFKA_CONFLUENT_CLOUD_KEY }} | |
| TEST_KAFKA_CONFLUENT_CLOUD_SECRET: ${{ secrets.TEST_KAFKA_CONFLUENT_CLOUD_SECRET }} | |
| TEST_KAFKA_AZURE_EVENT_HUBS_CLOUD_HOST: ${{ secrets.TEST_KAFKA_AZURE_EVENT_HUBS_CLOUD_HOST }} | |
| TEST_KAFKA_AZURE_EVENT_HUBS_CLOUD_EVENTHUB_NAME: ${{ secrets.TEST_KAFKA_AZURE_EVENT_HUBS_CLOUD_EVENTHUB_NAME }} | |
| TEST_KAFKA_AZURE_EVENT_HUBS_CLOUD_CONNECTION_STRING: ${{ secrets.TEST_KAFKA_AZURE_EVENT_HUBS_CLOUD_CONNECTION_STRING }} | |
| TEST_S3_DATALAKE_CREDENTIALS: ${{ secrets.TEST_S3_DATALAKE_CREDENTIALS }} | |
| BIGQUERY_INTEGRATION_TEST_CREDENTIALS: ${{ secrets.BIGQUERY_INTEGRATION_TEST_CREDENTIALS }} | |
| SNOWPIPE_STREAMING_KEYPAIR_UNENCRYPTED_INTEGRATION_TEST_CREDENTIALS: ${{ secrets.SNOWPIPE_STREAMING_KEYPAIR_UNENCRYPTED_INTEGRATION_TEST_CREDENTIALS }} | |
| SNOWPIPE_STREAMING_KEYPAIR_ENCRYPTED_INTEGRATION_TEST_CREDENTIALS: ${{ secrets.SNOWPIPE_STREAMING_KEYPAIR_ENCRYPTED_INTEGRATION_TEST_CREDENTIALS }} | |
| SNOWFLAKE_PRIVILEGE_INTEGRATION_TEST_CREDENTIALS: ${{ secrets.SNOWFLAKE_PRIVILEGE_INTEGRATION_TEST_CREDENTIALS }} | |
| RSERVER_PROCESSOR_ENABLE_CONCURRENT_STORE: "true" | |
| DOCKER_REGISTRY_MIRROR_USERNAME: ${{ vars.DOCKER_REGISTRY_MIRROR_USERNAME }} | |
| DOCKER_REGISTRY_MIRROR_PASSWORD: ${{ secrets.DOCKER_REGISTRY_MIRROR_PASSWORD }} | |
| DOCKER_REGISTRY_MIRROR: ${{ vars.DOCKER_REGISTRY_MIRROR }} | |
| RACE_ENABLED: ${{ matrix.race || false }} | |
| RSERVER_OAUTH_TEST_CREDENTIALS: ${{ secrets.RSERVER_OAUTH_TEST_CREDENTIALS }} | |
| FORCE_RUN_INTEGRATION_TESTS: "true" | |
| run: make test exclude="${{ matrix.exclude }}" package=${{ matrix.package }} | |
| - name: Sanitize name for Artifact | |
| run: | | |
| name=$(echo -n "${{ matrix.package }}" | sed -e 's/[ \t:\/\\"<>|*?]/-/g' -e 's/--*/-/g') | |
| echo "ARTIFACT_NAME=$name" >> $GITHUB_ENV | |
| - name: Upload coverage report | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: ${{ env.ARTIFACT_NAME }}-unit | |
| path: coverage.txt | |
| coverage: | |
| name: Coverage | |
| runs-on: ubuntu-latest | |
| needs: | |
| - warehouse-integration | |
| - unit | |
| - package-unit | |
| steps: | |
| - name: Harden the runner (Audit all outbound calls) | |
| uses: step-security/harden-runner@f4a75cfd619ee5ce8d5b864b0d183aff3c69b55a # v2.13.1 | |
| with: | |
| egress-policy: audit | |
| - name: Checkout | |
| uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 | |
| - name: Setup Go | |
| uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 | |
| with: | |
| go-version-file: 'go.mod' | |
| - name: Download coverage reports | |
| uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 | |
| - name: Merge Coverage | |
| run: | | |
| go install github.com/wadey/gocovmerge@latest | |
| gocovmerge */coverage.txt > coverage.txt | |
| - uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 | |
| with: | |
| fail_ci_if_error: true | |
| files: ./coverage.txt | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| all-green: | |
| name: All | |
| if: always() | |
| runs-on: ubuntu-latest | |
| needs: | |
| - integration | |
| - warehouse-integration | |
| - unit | |
| - package-unit | |
| steps: | |
| - name: Harden the runner (Audit all outbound calls) | |
| uses: step-security/harden-runner@f4a75cfd619ee5ce8d5b864b0d183aff3c69b55a # v2.13.1 | |
| with: | |
| egress-policy: audit | |
| - uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe # v1.2.2 | |
| with: | |
| jobs: ${{ toJSON(needs) }} |