main-ci.yml

name: Main CI Pipeline

on:
  pull_request:
    branches: ["main"]
  merge_group:
    branches: ["main"]
    types: [checks_requested]
  push:
    branches: ["main"]

permissions:
  actions: write
  contents: write
  pull-requests: write
  checks: write
  statuses: write

concurrency:
  group: main-ci-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

jobs:
  pre_job:
    name: Skip Duplicate Checks
    runs-on: depot-ubuntu-24.04
    outputs:
      should_skip: ${{ steps.skip_check.outputs.should_skip || 'false' }}
    steps:
      - id: skip_check
        continue-on-error: true
        uses: fkirc/skip-duplicate-actions@f75f66ce1886f00957d99748a42c724f4330bdcf # v5.3.1
        with:
          cancel_others: 'true'
          concurrent_skipping: same_content_newer

  # Check which paths were changed to determine which tests to run
  check-changes:
    name: Check Changed Files
    needs: pre_job
    if: needs.pre_job.outputs.should_skip != 'true'
    runs-on: depot-ubuntu-24.04
    outputs:
      api-changed: ${{ steps.changes.outputs.api }}
      cli-changed: ${{ steps.changes.outputs.cli }}
      e2e-changed: ${{ steps.changes.outputs.e2e }}
      web-changed: ${{ steps.changes.outputs.web }}
      vdb-changed: ${{ steps.changes.outputs.vdb }}
      migration-changed: ${{ steps.changes.outputs.migration }}
    steps:
      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
      - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
        id: changes
        with:
          filters: |
            api:
              - 'api/**'
              - '.github/workflows/api-tests.yml'
              - 'docker/.env.example'
              - 'docker/envs/middleware.env.example'
              - 'docker/docker-compose.middleware.yaml'
              - 'docker/docker-compose-template.yaml'
              - 'docker/generate_docker_compose'
              - 'docker/ssrf_proxy/**'
              - 'docker/volumes/sandbox/conf/**'
            cli:
              - 'cli/**'
              - 'packages/tsconfig/**'
              - 'package.json'
              - 'pnpm-lock.yaml'
              - 'pnpm-workspace.yaml'
              - 'eslint.config.mjs'
              - '.npmrc'
              - '.nvmrc'
              - '.github/workflows/cli-tests.yml'
              - '.github/workflows/cli-docker-build.yml'
              - '.github/actions/setup-web/**'
            web:
              - 'web/**'
              - 'packages/**'
              - 'package.json'
              - 'pnpm-lock.yaml'
              - 'pnpm-workspace.yaml'
              - '.nvmrc'
              - '.github/workflows/web-tests.yml'
              - '.github/actions/setup-web/**'
            e2e:
              - 'api/**'
              - 'api/pyproject.toml'
              - 'api/uv.lock'
              - 'e2e/**'
              - 'web/**'
              - 'packages/**'
              - 'package.json'
              - 'pnpm-lock.yaml'
              - 'pnpm-workspace.yaml'
              - '.nvmrc'
              - 'docker/docker-compose.middleware.yaml'
              - 'docker/envs/middleware.env.example'
              - '.github/workflows/web-e2e.yml'
              - '.github/actions/setup-web/**'
            vdb:
              - 'api/core/rag/datasource/**'
              - 'api/tests/integration_tests/vdb/**'
              - 'api/conftest.py'
              - 'api/tests/pytest_dify.py'
              - 'api/providers/vdb/*/tests/**'
              - '.github/workflows/vdb-tests.yml'
              - 'docker/.env.example'
              - 'docker/envs/middleware.env.example'
              - 'docker/docker-compose.pytest.ports.yaml'
              - 'docker/docker-compose.yaml'
              - 'docker/docker-compose-template.yaml'
              - 'docker/generate_docker_compose'
              - 'docker/certbot/**'
              - 'docker/couchbase-server/**'
              - 'docker/elasticsearch/**'
              - 'docker/iris/**'
              - 'docker/nginx/**'
              - 'docker/pgvector/**'
              - 'docker/ssrf_proxy/**'
              - 'docker/startupscripts/**'
              - 'docker/tidb/**'
              - 'docker/volumes/**'
              - 'api/uv.lock'
              - 'api/pyproject.toml'
            migration:
              - 'api/migrations/**'
              - 'api/.env.example'
              - '.github/workflows/db-migration-test.yml'
              - 'docker/.env.example'
              - 'docker/envs/middleware.env.example'
              - 'docker/docker-compose.middleware.yaml'
              - 'docker/docker-compose-template.yaml'
              - 'docker/generate_docker_compose'
              - 'docker/ssrf_proxy/**'
              - 'docker/volumes/sandbox/conf/**'

  # Run tests in parallel while always emitting stable required checks.
  api-tests-run:
    name: Run API Tests
    needs:
      - pre_job
      - check-changes
    if: needs.pre_job.outputs.should_skip != 'true' && needs.check-changes.outputs.api-changed == 'true'
    uses: ./.github/workflows/api-tests.yml
    secrets: inherit

  api-tests-skip:
    name: Skip API Tests
    needs:
      - pre_job
      - check-changes
    if: needs.pre_job.outputs.should_skip != 'true' && needs.check-changes.outputs.api-changed != 'true'
    runs-on: depot-ubuntu-24.04
    steps:
      - name: Report skipped API tests
        run: echo "No API-related changes detected; skipping API tests."

  api-tests:
    name: API Tests
    if: ${{ always() }}
    needs:
      - pre_job
      - check-changes
      - api-tests-run
      - api-tests-skip
    runs-on: depot-ubuntu-24.04
    steps:
      - name: Finalize API Tests status
        env:
          SHOULD_SKIP_WORKFLOW: ${{ needs.pre_job.outputs.should_skip }}
          TESTS_CHANGED: ${{ needs.check-changes.outputs.api-changed }}
          RUN_RESULT: ${{ needs.api-tests-run.result }}
          SKIP_RESULT: ${{ needs.api-tests-skip.result }}
        run: |
          if [[ "$SHOULD_SKIP_WORKFLOW" == 'true' ]]; then
            echo "API tests were skipped because this workflow run duplicated a successful or newer run."
            exit 0
          fi

          if [[ "$TESTS_CHANGED" == 'true' ]]; then
            if [[ "$RUN_RESULT" == 'success' ]]; then
              echo "API tests ran successfully."
              exit 0
            fi

            echo "API tests were required but finished with result: $RUN_RESULT" >&2
            exit 1
          fi

          if [[ "$SKIP_RESULT" == 'success' ]]; then
            echo "API tests were skipped because no API-related files changed."
            exit 0
          fi

          echo "API tests were not required, but the skip job finished with result: $SKIP_RESULT" >&2
          exit 1

  cli-tests-run:
    name: Run CLI Tests
    needs:
      - pre_job
      - check-changes
    if: needs.pre_job.outputs.should_skip != 'true' && needs.check-changes.outputs.cli-changed == 'true'
    uses: ./.github/workflows/cli-tests.yml
    secrets: inherit

  cli-tests-skip:
    name: Skip CLI Tests
    needs:
      - pre_job
      - check-changes
    if: needs.pre_job.outputs.should_skip != 'true' && needs.check-changes.outputs.cli-changed != 'true'
    runs-on: depot-ubuntu-24.04
    steps:
      - name: Report skipped CLI tests
        run: echo "No CLI-related changes detected; skipping CLI tests."

  cli-tests:
    name: CLI Tests
    if: ${{ always() }}
    needs:
      - pre_job
      - check-changes
      - cli-tests-run
      - cli-tests-skip
    runs-on: depot-ubuntu-24.04
    steps:
      - name: Finalize CLI Tests status
        env:
          SHOULD_SKIP_WORKFLOW: ${{ needs.pre_job.outputs.should_skip }}
          TESTS_CHANGED: ${{ needs.check-changes.outputs.cli-changed }}
          RUN_RESULT: ${{ needs.cli-tests-run.result }}
          SKIP_RESULT: ${{ needs.cli-tests-skip.result }}
        run: |
          if [[ "$SHOULD_SKIP_WORKFLOW" == 'true' ]]; then
            echo "CLI tests were skipped because this workflow run duplicated a successful or newer run."
            exit 0
          fi

          if [[ "$TESTS_CHANGED" == 'true' ]]; then
            if [[ "$RUN_RESULT" == 'success' ]]; then
              echo "CLI tests ran successfully."
              exit 0
            fi

            echo "CLI tests were required but finished with result: $RUN_RESULT" >&2
            exit 1
          fi

          if [[ "$SKIP_RESULT" == 'success' ]]; then
            echo "CLI tests were skipped because no CLI-related files changed."
            exit 0
          fi

          echo "CLI tests were not required, but the skip job finished with result: $SKIP_RESULT" >&2
          exit 1

  web-tests-run:
    name: Run Web Tests
    needs:
      - pre_job
      - check-changes
    if: needs.pre_job.outputs.should_skip != 'true' && needs.check-changes.outputs.web-changed == 'true'
    uses: ./.github/workflows/web-tests.yml
    secrets: inherit

  web-tests-skip:
    name: Skip Web Tests
    needs:
      - pre_job
      - check-changes
    if: needs.pre_job.outputs.should_skip != 'true' && needs.check-changes.outputs.web-changed != 'true'
    runs-on: depot-ubuntu-24.04
    steps:
      - name: Report skipped web tests
        run: echo "No web-related changes detected; skipping web tests."

  web-tests:
    name: Web Tests
    if: ${{ always() }}
    needs:
      - pre_job
      - check-changes
      - web-tests-run
      - web-tests-skip
    runs-on: depot-ubuntu-24.04
    steps:
      - name: Finalize Web Tests status
        env:
          SHOULD_SKIP_WORKFLOW: ${{ needs.pre_job.outputs.should_skip }}
          TESTS_CHANGED: ${{ needs.check-changes.outputs.web-changed }}
          RUN_RESULT: ${{ needs.web-tests-run.result }}
          SKIP_RESULT: ${{ needs.web-tests-skip.result }}
        run: |
          if [[ "$SHOULD_SKIP_WORKFLOW" == 'true' ]]; then
            echo "Web tests were skipped because this workflow run duplicated a successful or newer run."
            exit 0
          fi

          if [[ "$TESTS_CHANGED" == 'true' ]]; then
            if [[ "$RUN_RESULT" == 'success' ]]; then
              echo "Web tests ran successfully."
              exit 0
            fi

            echo "Web tests were required but finished with result: $RUN_RESULT" >&2
            exit 1
          fi

          if [[ "$SKIP_RESULT" == 'success' ]]; then
            echo "Web tests were skipped because no web-related files changed."
            exit 0
          fi

          echo "Web tests were not required, but the skip job finished with result: $SKIP_RESULT" >&2
          exit 1

  web-e2e-run:
    name: Run Web Full-Stack E2E
    needs:
      - pre_job
      - check-changes
    if: needs.pre_job.outputs.should_skip != 'true' && needs.check-changes.outputs.e2e-changed == 'true'
    uses: ./.github/workflows/web-e2e.yml

  web-e2e-skip:
    name: Skip Web Full-Stack E2E
    needs:
      - pre_job
      - check-changes
    if: needs.pre_job.outputs.should_skip != 'true' && needs.check-changes.outputs.e2e-changed != 'true'
    runs-on: depot-ubuntu-24.04
    steps:
      - name: Report skipped web full-stack e2e
        run: echo "No E2E-related changes detected; skipping web full-stack E2E."

  web-e2e:
    name: Web Full-Stack E2E
    if: ${{ always() }}
    needs:
      - pre_job
      - check-changes
      - web-e2e-run
      - web-e2e-skip
    runs-on: depot-ubuntu-24.04
    steps:
      - name: Finalize Web Full-Stack E2E status
        env:
          SHOULD_SKIP_WORKFLOW: ${{ needs.pre_job.outputs.should_skip }}
          TESTS_CHANGED: ${{ needs.check-changes.outputs.e2e-changed }}
          RUN_RESULT: ${{ needs.web-e2e-run.result }}
          SKIP_RESULT: ${{ needs.web-e2e-skip.result }}
        run: |
          if [[ "$SHOULD_SKIP_WORKFLOW" == 'true' ]]; then
            echo "Web full-stack E2E was skipped because this workflow run duplicated a successful or newer run."
            exit 0
          fi

          if [[ "$TESTS_CHANGED" == 'true' ]]; then
            if [[ "$RUN_RESULT" == 'success' ]]; then
              echo "Web full-stack E2E ran successfully."
              exit 0
            fi

            echo "Web full-stack E2E was required but finished with result: $RUN_RESULT" >&2
            exit 1
          fi

          if [[ "$SKIP_RESULT" == 'success' ]]; then
            echo "Web full-stack E2E was skipped because no E2E-related files changed."
            exit 0
          fi

          echo "Web full-stack E2E was not required, but the skip job finished with result: $SKIP_RESULT" >&2
          exit 1

  style-check:
    name: Style Check
    needs: pre_job
    if: needs.pre_job.outputs.should_skip != 'true'
    uses: ./.github/workflows/style.yml

  vdb-tests-run:
    name: Run VDB Tests
    needs:
      - pre_job
      - check-changes
    if: needs.pre_job.outputs.should_skip != 'true' && needs.check-changes.outputs.vdb-changed == 'true'
    uses: ./.github/workflows/vdb-tests.yml

  vdb-tests-skip:
    name: Skip VDB Tests
    needs:
      - pre_job
      - check-changes
    if: needs.pre_job.outputs.should_skip != 'true' && needs.check-changes.outputs.vdb-changed != 'true'
    runs-on: depot-ubuntu-24.04
    steps:
      - name: Report skipped VDB tests
        run: echo "No VDB-related changes detected; skipping VDB tests."

  vdb-tests:
    name: VDB Tests
    if: ${{ always() }}
    needs:
      - pre_job
      - check-changes
      - vdb-tests-run
      - vdb-tests-skip
    runs-on: depot-ubuntu-24.04
    steps:
      - name: Finalize VDB Tests status
        env:
          SHOULD_SKIP_WORKFLOW: ${{ needs.pre_job.outputs.should_skip }}
          TESTS_CHANGED: ${{ needs.check-changes.outputs.vdb-changed }}
          RUN_RESULT: ${{ needs.vdb-tests-run.result }}
          SKIP_RESULT: ${{ needs.vdb-tests-skip.result }}
        run: |
          if [[ "$SHOULD_SKIP_WORKFLOW" == 'true' ]]; then
            echo "VDB tests were skipped because this workflow run duplicated a successful or newer run."
            exit 0
          fi

          if [[ "$TESTS_CHANGED" == 'true' ]]; then
            if [[ "$RUN_RESULT" == 'success' ]]; then
              echo "VDB tests ran successfully."
              exit 0
            fi

            echo "VDB tests were required but finished with result: $RUN_RESULT" >&2
            exit 1
          fi

          if [[ "$SKIP_RESULT" == 'success' ]]; then
            echo "VDB tests were skipped because no VDB-related files changed."
            exit 0
          fi

          echo "VDB tests were not required, but the skip job finished with result: $SKIP_RESULT" >&2
          exit 1

  db-migration-test-run:
    name: Run DB Migration Test
    needs:
      - pre_job
      - check-changes
    if: needs.pre_job.outputs.should_skip != 'true' && needs.check-changes.outputs.migration-changed == 'true'
    uses: ./.github/workflows/db-migration-test.yml

  db-migration-test-skip:
    name: Skip DB Migration Test
    needs:
      - pre_job
      - check-changes
    if: needs.pre_job.outputs.should_skip != 'true' && needs.check-changes.outputs.migration-changed != 'true'
    runs-on: depot-ubuntu-24.04
    steps:
      - name: Report skipped DB migration tests
        run: echo "No migration-related changes detected; skipping DB migration tests."

  db-migration-test:
    name: DB Migration Test
    if: ${{ always() }}
    needs:
      - pre_job
      - check-changes
      - db-migration-test-run
      - db-migration-test-skip
    runs-on: depot-ubuntu-24.04
    steps:
      - name: Finalize DB Migration Test status
        env:
          SHOULD_SKIP_WORKFLOW: ${{ needs.pre_job.outputs.should_skip }}
          TESTS_CHANGED: ${{ needs.check-changes.outputs.migration-changed }}
          RUN_RESULT: ${{ needs.db-migration-test-run.result }}
          SKIP_RESULT: ${{ needs.db-migration-test-skip.result }}
        run: |
          if [[ "$SHOULD_SKIP_WORKFLOW" == 'true' ]]; then
            echo "DB migration tests were skipped because this workflow run duplicated a successful or newer run."
            exit 0
          fi

          if [[ "$TESTS_CHANGED" == 'true' ]]; then
            if [[ "$RUN_RESULT" == 'success' ]]; then
              echo "DB migration tests ran successfully."
              exit 0
            fi

            echo "DB migration tests were required but finished with result: $RUN_RESULT" >&2
            exit 1
          fi

          if [[ "$SKIP_RESULT" == 'success' ]]; then
            echo "DB migration tests were skipped because no migration-related files changed."
            exit 0
          fi

          echo "DB migration tests were not required, but the skip job finished with result: $SKIP_RESULT" >&2
          exit 1