DE integration tests #138
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: "DE integration tests" | |
| on: | |
| schedule: | |
| - cron: '0 1 * * 0' | |
| issue_comment: | |
| types: [created] | |
| workflow_dispatch: | |
| inputs: | |
| pytest_filter: | |
| description: "Pytest -k filter expression (leave empty to run all DE tests)" | |
| required: false | |
| type: string | |
| default: "" | |
| pr_number: | |
| description: "PR number to test (leave empty for current branch)" | |
| required: false | |
| type: string | |
| default: "" | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| id-token: write | |
| concurrency: | |
| group: de-${{ github.event.issue.number || github.event.inputs.pr_number || 'main' }} | |
| cancel-in-progress: true | |
| jobs: | |
| scheduled-de-tests: | |
| name: DE tests (scheduled) | |
| if: github.event_name == 'schedule' | |
| runs-on: ubuntu-latest | |
| environment: azure | |
| steps: | |
| - name: Install mssql-python system dependencies | |
| run: sudo apt-get update && sudo apt-get install -y libltdl7 | |
| - uses: actions/checkout@v6 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| python-version: "3.13" | |
| - name: Install dependencies | |
| run: uv sync --all-extras --all-groups | |
| - name: Run DE integration tests | |
| env: | |
| FABRIC_TEST_AUTH: workload_identity | |
| FABRIC_TEST_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} | |
| FABRIC_TEST_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} | |
| FABRIC_TEST_WORKSPACE_NAME: adapter-ci | |
| FABRIC_TEST_LAKEHOUSE_NAME: cilh | |
| FABRIC_TEST_LIVY_SESSION_NAME: ${{ github.run_id }} | |
| FABRIC_TEST_THREADS: "4" | |
| run: | | |
| export FABRIC_TEST_FEDERATED_TOKEN_URL="${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=api://AzureADTokenExchange" | |
| export FABRIC_TEST_FEDERATED_TOKEN_HEADER="bearer ${ACTIONS_ID_TOKEN_REQUEST_TOKEN}" | |
| uv run pytest -ra -v --de | |
| - name: Upload test logs | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: logs-de-scheduled | |
| path: logs/ | |
| on-demand-de-tests: | |
| name: DE tests (on-demand) | |
| if: >- | |
| (github.event_name == 'workflow_dispatch' && github.event.inputs.pr_number != '') || | |
| (github.event_name == 'issue_comment' | |
| && github.event.issue.pull_request | |
| && (github.event.comment.body == '/test-de' || startsWith(github.event.comment.body, '/test-de ')) | |
| && contains(fromJSON('["OWNER","MEMBER","COLLABORATOR"]'), github.event.comment.author_association)) | |
| runs-on: ubuntu-latest | |
| environment: azure | |
| steps: | |
| - name: Resolve PR metadata | |
| id: pr | |
| uses: actions/github-script@v9 | |
| env: | |
| INPUT_PR_NUMBER: ${{ github.event.inputs.pr_number }} | |
| with: | |
| script: | | |
| const prNumber = context.eventName === 'issue_comment' | |
| ? context.issue.number | |
| : parseInt(process.env.INPUT_PR_NUMBER); | |
| if (!prNumber) { | |
| core.setFailed('No PR number provided'); | |
| return; | |
| } | |
| const { data: pr } = await github.rest.pulls.get({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| pull_number: prNumber, | |
| }); | |
| if (pr.head.repo.full_name !== `${context.repo.owner}/${context.repo.repo}`) { | |
| core.setFailed('Cannot run tests on PRs from forks'); | |
| return; | |
| } | |
| core.setOutput('number', prNumber); | |
| core.setOutput('head_ref', pr.head.ref); | |
| core.setOutput('head_sha', pr.head.sha); | |
| - name: Parse pytest filter | |
| id: filter | |
| uses: actions/github-script@v9 | |
| env: | |
| INPUT_PYTEST_FILTER: ${{ github.event.inputs.pytest_filter }} | |
| with: | |
| script: | | |
| let rawFilter = ''; | |
| if (context.eventName === 'issue_comment') { | |
| const body = context.payload.comment.body; | |
| const match = body.match(/^\/test-de\s+(.*)/); | |
| rawFilter = match ? match[1].split('\n')[0].trim() : ''; | |
| } else { | |
| rawFilter = process.env.INPUT_PYTEST_FILTER || ''; | |
| } | |
| const sanitized = rawFilter.replace(/[^a-zA-Z0-9 _()]/g, ''); | |
| core.setOutput('expression', sanitized); | |
| core.setOutput('has_filter', sanitized.length > 0 ? 'true' : 'false'); | |
| core.info(`Pytest filter: ${sanitized || '(none — running all DE tests)'}`); | |
| - name: Acknowledge comment | |
| if: github.event_name == 'issue_comment' | |
| uses: actions/github-script@v9 | |
| with: | |
| script: | | |
| await github.rest.reactions.createForIssueComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| comment_id: context.payload.comment.id, | |
| content: 'rocket', | |
| }); | |
| - name: Install mssql-python system dependencies | |
| run: sudo apt-get update && sudo apt-get install -y libltdl7 | |
| - uses: actions/checkout@v6 | |
| with: | |
| ref: ${{ steps.pr.outputs.head_sha }} | |
| persist-credentials: false | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| python-version: "3.13" | |
| - name: Install dependencies | |
| run: uv sync --all-extras --all-groups | |
| - name: Run DE integration tests | |
| id: tests | |
| env: | |
| FABRIC_TEST_AUTH: workload_identity | |
| FABRIC_TEST_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} | |
| FABRIC_TEST_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} | |
| FABRIC_TEST_WORKSPACE_NAME: adapter-ci | |
| FABRIC_TEST_LAKEHOUSE_NAME: cilh | |
| FABRIC_TEST_LIVY_SESSION_NAME: ${{ github.run_id }} | |
| FABRIC_TEST_THREADS: "4" | |
| PYTEST_FILTER: ${{ steps.filter.outputs.expression }} | |
| run: | | |
| export FABRIC_TEST_FEDERATED_TOKEN_URL="${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=api://AzureADTokenExchange" | |
| export FABRIC_TEST_FEDERATED_TOKEN_HEADER="bearer ${ACTIONS_ID_TOKEN_REQUEST_TOKEN}" | |
| if [ -n "$PYTEST_FILTER" ]; then | |
| uv run pytest -ra -v --de -k "$PYTEST_FILTER" | |
| else | |
| uv run pytest -ra -v --de | |
| fi | |
| - name: Upload test logs | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: logs-de-on-demand-${{ steps.pr.outputs.number }} | |
| path: logs/ | |
| - name: Report results | |
| if: always() | |
| uses: actions/github-script@v9 | |
| env: | |
| FILTER_EXPR: ${{ steps.filter.outputs.expression }} | |
| TEST_OUTCOME: ${{ steps.tests.outcome }} | |
| with: | |
| script: | | |
| const prNumber = ${{ steps.pr.outputs.number }}; | |
| const success = process.env.TEST_OUTCOME === 'success'; | |
| const filter = process.env.FILTER_EXPR || '(all DE tests)'; | |
| const runUrl = `${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`; | |
| const emoji = success ? ':white_check_mark:' : ':x:'; | |
| const status = success ? 'passed' : 'failed'; | |
| const body = [ | |
| `${emoji} **DE integration tests ${status}**`, | |
| ``, | |
| `**Filter:** \`${filter}\``, | |
| `**Run:** ${runUrl}`, | |
| ].join('\n'); | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: prNumber, | |
| body, | |
| }); | |
| dispatch-de-tests: | |
| name: DE tests (dispatch, no PR) | |
| if: github.event_name == 'workflow_dispatch' && github.event.inputs.pr_number == '' | |
| runs-on: ubuntu-latest | |
| environment: azure | |
| steps: | |
| - name: Install mssql-python system dependencies | |
| run: sudo apt-get update && sudo apt-get install -y libltdl7 | |
| - uses: actions/checkout@v6 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| python-version: "3.13" | |
| - name: Install dependencies | |
| run: uv sync --all-extras --all-groups | |
| - name: Run DE integration tests | |
| env: | |
| FABRIC_TEST_AUTH: workload_identity | |
| FABRIC_TEST_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} | |
| FABRIC_TEST_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} | |
| FABRIC_TEST_WORKSPACE_NAME: adapter-ci | |
| FABRIC_TEST_LAKEHOUSE_NAME: cilh | |
| FABRIC_TEST_LIVY_SESSION_NAME: ${{ github.run_id }} | |
| FABRIC_TEST_THREADS: "4" | |
| PYTEST_FILTER: ${{ github.event.inputs.pytest_filter }} | |
| run: | | |
| export FABRIC_TEST_FEDERATED_TOKEN_URL="${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=api://AzureADTokenExchange" | |
| export FABRIC_TEST_FEDERATED_TOKEN_HEADER="bearer ${ACTIONS_ID_TOKEN_REQUEST_TOKEN}" | |
| if [ -n "$PYTEST_FILTER" ]; then | |
| uv run pytest -ra -v --de -k "$PYTEST_FILTER" | |
| else | |
| uv run pytest -ra -v --de | |
| fi | |
| - name: Upload test logs | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: logs-de-dispatch | |
| path: logs/ |