Connector Tests Trigger #16551
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Connector Tests Trigger | |
| on: | |
| workflow_run: | |
| workflows: ["Python Build", "Metadata Ingestion"] | |
| types: [completed] | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.workflow_run.name }}-${{ github.event.workflow_run.head_branch }} | |
| cancel-in-progress: true | |
| jobs: | |
| dispatch: | |
| runs-on: ubuntu-latest | |
| if: | | |
| github.event.workflow_run.conclusion == 'success' && | |
| github.event.workflow_run.head_repository.full_name == github.repository | |
| permissions: | |
| actions: read | |
| deployments: read | |
| steps: | |
| - name: Generate GitHub App token | |
| id: app-token | |
| uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1 | |
| with: | |
| app-id: ${{ secrets.CONNECTOR_TESTS_APP_ID }} | |
| private-key: ${{ secrets.CONNECTOR_TESTS_APP_PRIVATE_KEY }} | |
| - name: Check BOTH workflows passed for this SHA | |
| id: check-both | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| SHA="${{ github.event.workflow_run.head_sha }}" | |
| REPO="${{ github.repository }}" | |
| WB=$(gh api "/repos/${REPO}/actions/workflows/python-build-pages.yml/runs?head_sha=${SHA}&per_page=1" \ | |
| --jq '.workflow_runs[0].conclusion // empty') | |
| MI_RUNS=$(gh api "/repos/${REPO}/actions/workflows/metadata-ingestion.yml/runs?head_sha=${SHA}&per_page=1") | |
| MI=$(echo "$MI_RUNS" | jq -r '.workflow_runs[0].conclusion // empty') | |
| MI_RUN_ID=$(echo "$MI_RUNS" | jq -r '.workflow_runs[0].id // empty') | |
| echo "mi_run_id=${MI_RUN_ID}" >> "$GITHUB_OUTPUT" | |
| echo "Python Build: $WB, Metadata Ingestion: $MI" | |
| if [ "$WB" = "success" ] && [ "$MI" = "success" ]; then | |
| echo "both_passed=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "Not both passed yet (WB=$WB, MI=$MI) — other trigger will handle it. If one of the upstream job failed, then connector tests will not run for this SHA." | |
| echo "both_passed=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Get Cloudflare URL | |
| if: steps.check-both.outputs.both_passed == 'true' | |
| id: cloudflare | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| SHA="${{ github.event.workflow_run.head_sha }}" | |
| REPO="${{ github.repository }}" | |
| # The wheels build publishes to Cloudflare Pages, which registers a GitHub | |
| # deployment whose .environment is the generic Pages environment ("preview" on | |
| # PRs, "production" on master) — it does NOT contain "wheels". The wheels project | |
| # is only identifiable by its deployment URL (*.datahub-wheels.pages.dev). The | |
| # same SHA also carries unrelated docs/frontend Pages deployments, so we scan all | |
| # deployments for the SHA and pick the one whose success-status URL is the wheels | |
| # site. | |
| DEPLOY_URL="" | |
| for ID in $(gh api "/repos/${REPO}/deployments?sha=${SHA}&per_page=30" --jq '.[].id'); do | |
| URL=$(gh api "/repos/${REPO}/deployments/${ID}/statuses" \ | |
| --jq '[.[] | select(.state == "success") | .environment_url][0] // empty') | |
| case "$URL" in | |
| *wheels*) DEPLOY_URL="$URL"; break ;; | |
| esac | |
| done | |
| if [ -z "$DEPLOY_URL" ]; then | |
| echo "::error::No Cloudflare wheels deployment (*.datahub-wheels.pages.dev) found for SHA ${SHA}. Re-run the Python Build workflow." | |
| exit 1 | |
| fi | |
| echo "url=${DEPLOY_URL}" >> "$GITHUB_OUTPUT" | |
| echo "Cloudflare URL: ${DEPLOY_URL}" | |
| - name: Get PR info | |
| if: steps.check-both.outputs.both_passed == 'true' | |
| id: pr-info | |
| run: | | |
| PR_NUMBER="${{ github.event.workflow_run.pull_requests[0].number }}" | |
| if [ -z "$PR_NUMBER" ]; then | |
| echo "No open PR found — skipping" | |
| echo "has_pr=false" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "has_pr=true" >> "$GITHUB_OUTPUT" | |
| echo "pr_number=$PR_NUMBER" >> "$GITHUB_OUTPUT" | |
| echo "Found PR #${PR_NUMBER}" | |
| fi | |
| - name: Check for skip label | |
| if: steps.check-both.outputs.both_passed == 'true' && steps.pr-info.outputs.has_pr == 'true' | |
| id: check-skip | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| APP_TOKEN: ${{ steps.app-token.outputs.token }} | |
| REPO: ${{ github.repository }} | |
| PR_NUMBER: ${{ steps.pr-info.outputs.pr_number }} | |
| run: | | |
| # Check if PR has the skip-connector-tests label | |
| HAS_LABEL=$(gh api "/repos/${REPO}/pulls/${PR_NUMBER}" \ | |
| --jq '[.labels[].name] | any(. == "skip-connector-tests")' 2>/dev/null || echo "false") | |
| if [ "$HAS_LABEL" != "true" ]; then | |
| echo "skip=false" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| # Find who added the label via timeline API | |
| ADDER=$(gh api --paginate "/repos/${REPO}/issues/${PR_NUMBER}/timeline" \ | |
| --jq '[.[] | select(.event == "labeled" and .label.name == "skip-connector-tests")] | last | .actor.login // empty') | |
| if [ -z "$ADDER" ]; then | |
| echo "::warning::skip-connector-tests label found but could not determine who added it — running tests" | |
| echo "skip=false" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| # Check if the label adder is a member of the datahub-project org | |
| MEMBER_STATUS=$(GH_TOKEN="$APP_TOKEN" gh api "/orgs/datahub-project/members/${ADDER}" \ | |
| --silent -i 2>&1 | head -1 | grep -o '[0-9]\{3\}' | head -1) | |
| if [ "$MEMBER_STATUS" = "204" ]; then | |
| echo "PR #${PR_NUMBER} has skip-connector-tests label added by ${ADDER} (org member) — skipping tests" | |
| echo "skip=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "::warning::skip-connector-tests label added by ${ADDER} who is not a datahub-project org member — ignoring label, running tests" | |
| echo "skip=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Check if connector-relevant files changed | |
| if: steps.check-both.outputs.both_passed == 'true' && steps.pr-info.outputs.has_pr == 'true' && steps.check-skip.outputs.skip != 'true' | |
| id: path-check | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| REPO: ${{ github.repository }} | |
| PR_NUMBER: ${{ steps.pr-info.outputs.pr_number }} | |
| run: | | |
| changed_files=$(gh api "repos/${REPO}/pulls/${PR_NUMBER}/files" \ | |
| --paginate --jq '.[].filename') | |
| # Filter to only metadata-ingestion changes | |
| mi_files=$(echo "$changed_files" | grep '^metadata-ingestion/' || true) | |
| if [ -z "$mi_files" ]; then | |
| echo "No metadata-ingestion changes — skipping connector tests" | |
| echo "has_connector_changes=false" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| # Strip out files that cannot affect connectors | |
| connector_relevant_files=$(echo "$mi_files" | grep -v '^metadata-ingestion/tests/' | grep -v '\.md$' || true) | |
| if [ -z "$connector_relevant_files" ]; then | |
| echo "All metadata-ingestion changes are tests or docs only — skipping connector tests" | |
| echo "has_connector_changes=false" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "Connector-relevant changes detected:" | |
| echo "$connector_relevant_files" | |
| echo "has_connector_changes=true" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Determine target connectors | |
| if: >- | |
| steps.check-both.outputs.both_passed == 'true' && | |
| steps.pr-info.outputs.has_pr == 'true' && | |
| steps.check-skip.outputs.skip != 'true' && | |
| steps.path-check.outputs.has_connector_changes != 'false' | |
| id: connectors | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| REPO: ${{ github.repository }} | |
| MI_RUN_ID: ${{ steps.check-both.outputs.mi_run_id }} | |
| run: | | |
| if ! gh run download "$MI_RUN_ID" --repo "$REPO" --name "ci-decisions" --dir ci-decisions 2>/dev/null; then | |
| echo "::warning::Could not download ci-decisions artifact — running all connector tests" | |
| echo "connectors=" >> "$GITHUB_OUTPUT" | |
| { | |
| echo "### Connector Tests Decision" | |
| echo "**Run all (fallback)** — ci-decisions artifact not found." | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| exit 0 | |
| fi | |
| RUN_ALL=$(jq -r '.run_all_integration' ci-decisions/ci-decisions.json) | |
| if [ "$RUN_ALL" = "true" ]; then | |
| echo "run_all_integration=true — running all connector tests" | |
| echo "connectors=" >> "$GITHUB_OUTPUT" | |
| { | |
| echo "### Connector Tests Decision" | |
| echo "**Run all** — shared/non-connector files changed." | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| exit 0 | |
| fi | |
| # Static mapping: datahub integration test dir name → connector-tests connector name | |
| # Update this when acryldata/connector-tests adds or removes connectors | |
| # TODO: Remove this mapping once both of the following are true: | |
| # (i) ci-decisions.json surfaces the canonical connector name (not the test-dir name) | |
| # (ii) acryldata/connector-tests organises its tests in directories using those same names | |
| CONNECTORS=$(jq -r '.test_matrix[].connector' ci-decisions/ci-decisions.json 2>/dev/null | \ | |
| while IFS= read -r name; do | |
| case "$name" in | |
| athena) echo "athena" ;; | |
| azure_data_factory) echo "azure_data_factory" ;; | |
| bigquery_v2) echo "bigquery" ;; | |
| fabric) echo "fabric_onelake" ;; | |
| fivetran) echo "fivetran" ;; | |
| looker) echo "looker" ;; | |
| powerbi) echo "powerbi" ;; | |
| redshift) echo "redshift" ;; | |
| sigma) echo "sigma" ;; | |
| snowflake) echo "snowflake" ;; | |
| tableau) echo "tableau" ;; | |
| unity) echo "databricks" ;; | |
| esac | |
| done | sort -u | paste -sd ',') | |
| if [ -z "$CONNECTORS" ]; then | |
| echo "No connector-tests connectors matched — skipping dispatch" | |
| echo "connectors=" >> "$GITHUB_OUTPUT" | |
| echo "skip=true" >> "$GITHUB_OUTPUT" | |
| { | |
| echo "### Connector Tests Decision" | |
| echo "**Skipped** — changes do not affect any connector-tests connector." | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| else | |
| echo "Target connectors: $CONNECTORS" | |
| echo "connectors=$CONNECTORS" >> "$GITHUB_OUTPUT" | |
| echo "skip=false" >> "$GITHUB_OUTPUT" | |
| { | |
| echo "### Connector Tests Decision" | |
| echo "**Selective** — running: \`$CONNECTORS\`" | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| fi | |
| - name: Create skipped check run | |
| if: >- | |
| steps.check-both.outputs.both_passed == 'true' && | |
| steps.pr-info.outputs.has_pr == 'true' && | |
| (steps.check-skip.outputs.skip == 'true' || steps.path-check.outputs.has_connector_changes == 'false' || steps.connectors.outputs.skip == 'true') | |
| env: | |
| GH_TOKEN: ${{ steps.app-token.outputs.token }} | |
| REPO: ${{ github.repository }} | |
| SHA: ${{ github.event.workflow_run.head_sha }} | |
| run: | | |
| if [ "${{ steps.check-skip.outputs.skip }}" = "true" ]; then | |
| REASON="Skipped via skip-connector-tests label on the PR." | |
| elif [ "${{ steps.connectors.outputs.skip }}" = "true" ]; then | |
| REASON="Skipped — PR changes do not affect any connector tested by the connector-tests suite." | |
| else | |
| REASON="Skipped because PR changes are limited to tests or docs in metadata-ingestion." | |
| fi | |
| jq -nc \ | |
| --arg sha "$SHA" \ | |
| --arg completed_at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ | |
| --arg summary "$REASON" \ | |
| '{name: "connector-tests", head_sha: $sha, status: "completed", conclusion: "skipped", completed_at: $completed_at, output: {title: "Connector tests skipped", summary: $summary}}' \ | |
| | gh api "/repos/${REPO}/check-runs" --input - | |
| echo "Created skipped check run — ${REASON}" | |
| - name: Create pending check run | |
| if: >- | |
| steps.check-both.outputs.both_passed == 'true' && | |
| steps.pr-info.outputs.has_pr == 'true' && | |
| steps.check-skip.outputs.skip != 'true' && | |
| steps.path-check.outputs.has_connector_changes != 'false' && | |
| steps.connectors.outputs.skip != 'true' | |
| id: check-run | |
| env: | |
| GH_TOKEN: ${{ steps.app-token.outputs.token }} | |
| REPO: ${{ github.repository }} | |
| SHA: ${{ github.event.workflow_run.head_sha }} | |
| run: | | |
| CHECK_RUN_ID=$(jq -nc \ | |
| --arg sha "$SHA" \ | |
| --arg started_at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ | |
| '{name: "connector-tests", head_sha: $sha, status: "in_progress", started_at: $started_at, output: {title: "Connector tests in progress", summary: "Connector tests have been triggered and are running..."}}' \ | |
| | gh api "/repos/${REPO}/check-runs" --input - --jq '.id') | |
| echo "check_run_id=${CHECK_RUN_ID}" >> "$GITHUB_OUTPUT" | |
| echo "Created check run ${CHECK_RUN_ID}" | |
| - name: Dispatch to connector-tests | |
| if: >- | |
| steps.check-both.outputs.both_passed == 'true' && | |
| steps.pr-info.outputs.has_pr == 'true' && | |
| steps.check-skip.outputs.skip != 'true' && | |
| steps.path-check.outputs.has_connector_changes != 'false' && | |
| steps.connectors.outputs.skip != 'true' | |
| env: | |
| # Fine-grained PAT scoped to acryldata/connector-tests with Contents: R/W. | |
| # To rotate: generate a new token and update this repo secret. | |
| GH_TOKEN: ${{ secrets.CONNECTOR_TESTS_DISPATCH_TOKEN }} | |
| PR_NUMBER: ${{ steps.pr-info.outputs.pr_number }} | |
| COMMIT_SHA: ${{ github.event.workflow_run.head_sha }} | |
| SOURCE_REPO: ${{ github.repository }} | |
| WHEEL_URL: ${{ steps.cloudflare.outputs.url }} | |
| CHECK_RUN_ID: ${{ steps.check-run.outputs.check_run_id }} | |
| CONNECTORS: ${{ steps.connectors.outputs.connectors }} | |
| run: | | |
| jq -nc \ | |
| --arg event "oss-pr-connector-tests" \ | |
| --arg pr "$PR_NUMBER" \ | |
| --arg sha "$COMMIT_SHA" \ | |
| --arg repo "$SOURCE_REPO" \ | |
| --arg url "$WHEEL_URL" \ | |
| --arg check_run_id "$CHECK_RUN_ID" \ | |
| --arg connectors "$CONNECTORS" \ | |
| '{event_type: $event, client_payload: ({pr_number: $pr, sha: $sha, source_repo: $repo, wheel_url: $url, check_run_id: $check_run_id} + (if $connectors != "" then {connectors: $connectors} else {} end))}' \ | |
| | gh api "/repos/acryldata/connector-tests/dispatches" --input - | |
| if [ -n "$CONNECTORS" ]; then | |
| echo "Dispatched for PR #${PR_NUMBER} — connectors: ${CONNECTORS}" | |
| else | |
| echo "Dispatched for PR #${PR_NUMBER} — all connectors" | |
| fi | |
| - name: On failure — mark check run as failed | |
| if: (failure() || cancelled()) && steps.check-run.outputs.check_run_id != '' | |
| env: | |
| GH_TOKEN: ${{ steps.app-token.outputs.token }} | |
| CHECK_RUN_ID: ${{ steps.check-run.outputs.check_run_id }} | |
| REPO: ${{ github.repository }} | |
| run: | | |
| jq -nc \ | |
| --arg completed_at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ | |
| '{status: "completed", conclusion: "failure", completed_at: $completed_at, output: {title: "Failed to trigger connector tests", summary: "Infrastructure issue prevented connector tests from being dispatched."}}' \ | |
| | gh api "/repos/${REPO}/check-runs/${CHECK_RUN_ID}" -X PATCH --input - |