integration-tests-de.yml

---
name: "DE integration tests"
on:
  schedule:
    - cron: '0 1 * * 0'
  issue_comment:
    types: [created]
  workflow_dispatch:
    inputs:
      pytest_filter:
        description: "Pytest -k filter expression (leave empty to run all DE tests)"
        required: false
        type: string
        default: ""
      pr_number:
        description: "PR number to test (leave empty for current branch)"
        required: false
        type: string
        default: ""

permissions:
  contents: read
  pull-requests: write
  id-token: write

concurrency:
  group: de-${{ github.event.issue.number || github.event.inputs.pr_number || 'main' }}
  cancel-in-progress: true

jobs:
  scheduled-de-tests:
    name: DE tests (scheduled)
    if: github.event_name == 'schedule'
    runs-on: ubuntu-latest
    environment: azure
    steps:
      - name: Install mssql-python system dependencies
        run: sudo apt-get update && sudo apt-get install -y libltdl7

      - uses: actions/checkout@v6

      - name: Install uv
        uses: astral-sh/setup-uv@v7
        with:
          python-version: "3.13"

      - name: Install dependencies
        run: uv sync --all-extras --all-groups

      - name: Run DE integration tests
        env:
          FABRIC_TEST_AUTH: workload_identity
          FABRIC_TEST_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
          FABRIC_TEST_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
          FABRIC_TEST_WORKSPACE_NAME: adapter-ci
          FABRIC_TEST_LAKEHOUSE_NAME: cilh
          FABRIC_TEST_LIVY_SESSION_NAME: ${{ github.run_id }}
          FABRIC_TEST_THREADS: "4"
        run: |
          export FABRIC_TEST_FEDERATED_TOKEN_URL="${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=api://AzureADTokenExchange"
          export FABRIC_TEST_FEDERATED_TOKEN_HEADER="bearer ${ACTIONS_ID_TOKEN_REQUEST_TOKEN}"
          uv run pytest -ra -v --de

      - name: Upload test logs
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: logs-de-scheduled
          path: logs/

  on-demand-de-tests:
    name: DE tests (on-demand)
    if: >-
      (github.event_name == 'workflow_dispatch' && github.event.inputs.pr_number != '') ||
      (github.event_name == 'issue_comment'
       && github.event.issue.pull_request
       && (github.event.comment.body == '/test-de' || startsWith(github.event.comment.body, '/test-de '))
       && contains(fromJSON('["OWNER","MEMBER","COLLABORATOR"]'), github.event.comment.author_association))
    runs-on: ubuntu-latest
    environment: azure

    steps:
      - name: Resolve PR metadata
        id: pr
        uses: actions/github-script@v9
        env:
          INPUT_PR_NUMBER: ${{ github.event.inputs.pr_number }}
        with:
          script: |
            const prNumber = context.eventName === 'issue_comment'
              ? context.issue.number
              : parseInt(process.env.INPUT_PR_NUMBER);

            if (!prNumber) {
              core.setFailed('No PR number provided');
              return;
            }

            const { data: pr } = await github.rest.pulls.get({
              owner: context.repo.owner,
              repo: context.repo.repo,
              pull_number: prNumber,
            });

            if (pr.head.repo.full_name !== `${context.repo.owner}/${context.repo.repo}`) {
              core.setFailed('Cannot run tests on PRs from forks');
              return;
            }

            core.setOutput('number', prNumber);
            core.setOutput('head_ref', pr.head.ref);
            core.setOutput('head_sha', pr.head.sha);

      - name: Parse pytest filter
        id: filter
        uses: actions/github-script@v9
        env:
          INPUT_PYTEST_FILTER: ${{ github.event.inputs.pytest_filter }}
        with:
          script: |
            let rawFilter = '';
            if (context.eventName === 'issue_comment') {
              const body = context.payload.comment.body;
              const match = body.match(/^\/test-de\s+(.*)/);
              rawFilter = match ? match[1].split('\n')[0].trim() : '';
            } else {
              rawFilter = process.env.INPUT_PYTEST_FILTER || '';
            }

            const sanitized = rawFilter.replace(/[^a-zA-Z0-9 _()]/g, '');
            core.setOutput('expression', sanitized);
            core.setOutput('has_filter', sanitized.length > 0 ? 'true' : 'false');
            core.info(`Pytest filter: ${sanitized || '(none — running all DE tests)'}`);

      - name: Acknowledge comment
        if: github.event_name == 'issue_comment'
        uses: actions/github-script@v9
        with:
          script: |
            await github.rest.reactions.createForIssueComment({
              owner: context.repo.owner,
              repo: context.repo.repo,
              comment_id: context.payload.comment.id,
              content: 'rocket',
            });

      - name: Install mssql-python system dependencies
        run: sudo apt-get update && sudo apt-get install -y libltdl7

      - uses: actions/checkout@v6
        with:
          ref: ${{ steps.pr.outputs.head_sha }}
          persist-credentials: false

      - name: Install uv
        uses: astral-sh/setup-uv@v7
        with:
          python-version: "3.13"

      - name: Install dependencies
        run: uv sync --all-extras --all-groups

      - name: Run DE integration tests
        id: tests
        env:
          FABRIC_TEST_AUTH: workload_identity
          FABRIC_TEST_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
          FABRIC_TEST_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
          FABRIC_TEST_WORKSPACE_NAME: adapter-ci
          FABRIC_TEST_LAKEHOUSE_NAME: cilh
          FABRIC_TEST_LIVY_SESSION_NAME: ${{ github.run_id }}
          FABRIC_TEST_THREADS: "4"
          PYTEST_FILTER: ${{ steps.filter.outputs.expression }}
        run: |
          export FABRIC_TEST_FEDERATED_TOKEN_URL="${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=api://AzureADTokenExchange"
          export FABRIC_TEST_FEDERATED_TOKEN_HEADER="bearer ${ACTIONS_ID_TOKEN_REQUEST_TOKEN}"
          if [ -n "$PYTEST_FILTER" ]; then
            uv run pytest -ra -v --de -k "$PYTEST_FILTER"
          else
            uv run pytest -ra -v --de
          fi

      - name: Upload test logs
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: logs-de-on-demand-${{ steps.pr.outputs.number }}
          path: logs/

      - name: Report results
        if: always()
        uses: actions/github-script@v9
        env:
          FILTER_EXPR: ${{ steps.filter.outputs.expression }}
          TEST_OUTCOME: ${{ steps.tests.outcome }}
        with:
          script: |
            const prNumber = ${{ steps.pr.outputs.number }};
            const success = process.env.TEST_OUTCOME === 'success';
            const filter = process.env.FILTER_EXPR || '(all DE tests)';
            const runUrl = `${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`;

            const emoji = success ? ':white_check_mark:' : ':x:';
            const status = success ? 'passed' : 'failed';

            const body = [
              `${emoji} **DE integration tests ${status}**`,
              ``,
              `**Filter:** \`${filter}\``,
              `**Run:** ${runUrl}`,
            ].join('\n');

            await github.rest.issues.createComment({
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: prNumber,
              body,
            });

  dispatch-de-tests:
    name: DE tests (dispatch, no PR)
    if: github.event_name == 'workflow_dispatch' && github.event.inputs.pr_number == ''
    runs-on: ubuntu-latest
    environment: azure
    steps:
      - name: Install mssql-python system dependencies
        run: sudo apt-get update && sudo apt-get install -y libltdl7

      - uses: actions/checkout@v6

      - name: Install uv
        uses: astral-sh/setup-uv@v7
        with:
          python-version: "3.13"

      - name: Install dependencies
        run: uv sync --all-extras --all-groups

      - name: Run DE integration tests
        env:
          FABRIC_TEST_AUTH: workload_identity
          FABRIC_TEST_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
          FABRIC_TEST_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
          FABRIC_TEST_WORKSPACE_NAME: adapter-ci
          FABRIC_TEST_LAKEHOUSE_NAME: cilh
          FABRIC_TEST_LIVY_SESSION_NAME: ${{ github.run_id }}
          FABRIC_TEST_THREADS: "4"
          PYTEST_FILTER: ${{ github.event.inputs.pytest_filter }}
        run: |
          export FABRIC_TEST_FEDERATED_TOKEN_URL="${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=api://AzureADTokenExchange"
          export FABRIC_TEST_FEDERATED_TOKEN_HEADER="bearer ${ACTIONS_ID_TOKEN_REQUEST_TOKEN}"
          if [ -n "$PYTEST_FILTER" ]; then
            uv run pytest -ra -v --de -k "$PYTEST_FILTER"
          else
            uv run pytest -ra -v --de
          fi

      - name: Upload test logs
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: logs-de-dispatch
          path: logs/