ci-tests-e2e.yaml

# Description: End-to-end testing with Playwright across multiple browsers, deploys test reports to Cloudflare Pages
name: 'CI: Tests E2E'

on:
  push:
    branches: [main, master, core/*, desktop/*]
  pull_request:
    branches-ignore: [wip/*, draft/*, temp/*]
  merge_group:
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  changes:
    runs-on: ubuntu-latest
    permissions:
      contents: read
    outputs:
      should-run: ${{ steps.changes.outputs.should-run }}
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
      - id: changes
        uses: ./.github/actions/changes-filter

  setup:
    needs: changes
    if: ${{ needs.changes.outputs.should-run == 'true' }}
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
      - name: Setup frontend
        uses: ./.github/actions/setup-frontend
        with:
          include_build_step: true
        env:
          COLLECT_COVERAGE: 'true'

      # Upload only built dist/ (containerized test jobs will pnpm install without cache)
      - name: Upload built frontend
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
        with:
          name: frontend-dist
          path: dist/
          retention-days: 1

      - name: Build cloud frontend
        uses: ./.github/actions/build-cloud-frontend

  # Sharded chromium tests
  playwright-tests-chromium-sharded:
    needs: setup
    runs-on: ubuntu-latest
    timeout-minutes: 60
    container:
      image: ghcr.io/comfy-org/comfyui-ci-container:0.0.21
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
    permissions:
      contents: read
      packages: read
    strategy:
      fail-fast: false
      matrix:
        shardIndex: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]
        shardTotal: [16]
    steps:
      - name: Checkout repository
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
      - name: Download built frontend
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: frontend-dist
          path: dist/

      - name: Start ComfyUI server
        uses: ./.github/actions/start-comfyui-server

      - name: Install frontend deps
        run: pnpm install --frozen-lockfile

      # Run sharded tests (browsers pre-installed in container)
      - name: Run Playwright tests (Shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }})
        id: playwright
        run: pnpm exec playwright test --project=chromium --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
        env:
          PLAYWRIGHT_BLOB_OUTPUT_DIR: ./blob-report
          COLLECT_COVERAGE: 'true'

      - name: Upload blob report
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
        if: ${{ !cancelled() }}
        with:
          name: blob-report-chromium-${{ matrix.shardIndex }}
          path: blob-report/
          retention-days: 1

      - name: Upload shard coverage data
        if: always()
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
        with:
          name: e2e-coverage-shard-${{ matrix.shardIndex }}
          path: coverage/playwright/
          retention-days: 1
          if-no-files-found: warn

  playwright-tests:
    # Ideally, each shard runs test in 6 minutes, but allow up to 15 minutes
    timeout-minutes: 15
    needs: setup
    runs-on: ubuntu-latest
    container:
      image: ghcr.io/comfy-org/comfyui-ci-container:0.0.21
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
    permissions:
      contents: read
      packages: read
    strategy:
      fail-fast: false
      matrix:
        browser:
          [chromium-2x, chromium-0.5x, mobile-chrome, cloud, mobile-safari]
    steps:
      - name: Checkout repository
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
      - name: Download built frontend
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: ${{ (matrix.browser == 'cloud' || matrix.browser == 'mobile-safari') && 'frontend-dist-cloud' || 'frontend-dist' }}
          path: dist/

      - name: Start ComfyUI server
        uses: ./.github/actions/start-comfyui-server

      - name: Install frontend deps
        run: pnpm install --frozen-lockfile

      # Run tests (browsers pre-installed in container)
      - name: Run Playwright tests (${{ matrix.browser }})
        id: playwright
        run: pnpm exec playwright test --project=${{ matrix.browser }}
        env:
          PLAYWRIGHT_BLOB_OUTPUT_DIR: ./blob-report

      - name: Generate HTML and JSON reports
        if: always()
        run: |
          # Generate HTML report from blob
          pnpm exec playwright merge-reports --reporter=html ./blob-report
          # Generate JSON report separately with explicit output path
          PLAYWRIGHT_JSON_OUTPUT_NAME=playwright-report/report.json \
          pnpm exec playwright merge-reports --reporter=json ./blob-report

      - name: Upload Playwright report
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
        if: always()
        with:
          name: playwright-report-${{ matrix.browser }}
          path: ./playwright-report/
          retention-days: 30

  # Merge sharded test reports (no container needed - only runs CLI)
  merge-reports:
    needs: [changes, playwright-tests-chromium-sharded]
    runs-on: ubuntu-latest
    if: ${{ !cancelled() && needs.changes.outputs.should-run == 'true' }}
    steps:
      - name: Install pnpm
        uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9
        with:
          version: 10

      - name: Download blob reports
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          path: ./all-blob-reports
          pattern: blob-report-chromium-*
          merge-multiple: true

      - name: Merge into HTML Report
        run: |
          # Generate HTML report
          pnpm dlx @playwright/test merge-reports --reporter=html ./all-blob-reports
          # Generate JSON report separately with explicit output path
          PLAYWRIGHT_JSON_OUTPUT_NAME=playwright-report/report.json \
          pnpm dlx @playwright/test merge-reports --reporter=json ./all-blob-reports

      - name: Upload HTML report
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
        with:
          name: playwright-report-chromium
          path: ./playwright-report/
          retention-days: 30

  # Gate job — single required check that passes whether the matrix ran or was
  # skipped. Branch rulesets require this instead of the individual matrix-
  # expanded check names so PRs with no e2e-relevant changes aren't stuck.
  e2e-status:
    if: ${{ always() }}
    needs: [changes, playwright-tests-chromium-sharded, playwright-tests]
    runs-on: ubuntu-latest
    steps:
      - name: Check E2E results
        env:
          SHOULD_RUN: ${{ needs.changes.outputs.should-run }}
          SHARDED: ${{ needs.playwright-tests-chromium-sharded.result }}
          BROWSERS: ${{ needs.playwright-tests.result }}
        run: |
          [[ "$SHOULD_RUN" != "true" ]] && echo "E2E skipped" && exit 0
          [[ "$SHARDED" != "success" || "$BROWSERS" != "success" ]] && echo "E2E failed" && exit 1
          echo "E2E passed"

  # Records video only for spec files newly added in this PR, to keep cost bounded.
  playwright-video-new-tests:
    needs: setup
    if: ${{ github.event_name == 'pull_request' }}
    runs-on: ubuntu-latest
    timeout-minutes: 15
    container:
      image: ghcr.io/comfy-org/comfyui-ci-container:0.0.21
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
    permissions:
      contents: read
      packages: read
    outputs:
      has-new-tests: ${{ steps.detect.outputs.has-new-tests }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          fetch-depth: 0

      - name: Detect newly-added test spec files
        id: detect
        run: |
          git config --global --add safe.directory "$GITHUB_WORKSPACE"
          BASE_SHA="${{ github.event.pull_request.base.sha }}"
          NEW_FILES=$(git diff --name-only --diff-filter=A "$BASE_SHA"...HEAD -- browser_tests/tests | grep '\.spec\.ts$' || true)

          if [ -z "$NEW_FILES" ]; then
            echo "has-new-tests=false" >> "$GITHUB_OUTPUT"
          else
            echo "has-new-tests=true" >> "$GITHUB_OUTPUT"
          fi

          {
            echo "files<<EOF_NEW_SPEC_FILES"
            echo "$NEW_FILES"
            echo "EOF_NEW_SPEC_FILES"
          } >> "$GITHUB_OUTPUT"

          echo "Newly-added spec files:"
          echo "${NEW_FILES:-<none>}"

      - name: Download built frontend
        if: steps.detect.outputs.has-new-tests == 'true'
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: frontend-dist
          path: dist/

      - name: Start ComfyUI server
        if: steps.detect.outputs.has-new-tests == 'true'
        uses: ./.github/actions/start-comfyui-server

      - name: Install frontend deps
        if: steps.detect.outputs.has-new-tests == 'true'
        run: pnpm install --frozen-lockfile

      - name: Run new test(s) with video recording
        if: steps.detect.outputs.has-new-tests == 'true'
        env:
          RECORD_VIDEO: 'true'
          SLOW_MO: '1000'
          NEW_SPEC_FILES: ${{ steps.detect.outputs.files }}
        run: |
          FILES=$(echo "$NEW_SPEC_FILES" | tr '\n' ' ')
          pnpm exec playwright test --project=chromium $FILES

      - name: Upload new-test report (with embedded video)
        if: ${{ !cancelled() && steps.detect.outputs.has-new-tests == 'true' }}
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
        with:
          name: playwright-report-new-tests
          path: ./playwright-report/
          retention-days: 30
          if-no-files-found: warn

  #### BEGIN Deployment and commenting (PRs with direct secret access)
  # when using pull_request event, we have permission to comment directly
  # otherwise, we use workflow_run in ci-tests-e2e-forks.yaml

  # Post starting section into the unified PR report comment
  comment-on-pr-start:
    needs: changes
    runs-on: ubuntu-latest
    concurrency:
      group: pr-comment-${{ github.event.pull_request.number }}
      cancel-in-progress: false
    if: >-
      ${{
        needs.changes.outputs.should-run == 'true' &&
        github.event_name == 'pull_request' &&
        github.event.pull_request.head.repo.fork == false &&
        github.event.pull_request.user.login != 'dependabot[bot]'
      }}
    permissions:
      pull-requests: write
    steps:
      - name: Checkout repository
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Upsert playwright starting section into unified report
        uses: ./.github/actions/upsert-comment-section
        with:
          pr-number: ${{ github.event.pull_request.number }}
          section-name: playwright
          section-content: '## 🎭 Playwright: ⏳ Running...'
          token: ${{ github.token }}

  # Deploy and upsert final playwright section
  deploy-and-comment:
    needs:
      [changes, playwright-tests, merge-reports, playwright-video-new-tests]
    runs-on: ubuntu-latest
    concurrency:
      group: pr-comment-${{ github.event.pull_request.number }}
      cancel-in-progress: false
    if: >-
      ${{
        always() &&
        needs.changes.outputs.should-run == 'true' &&
        github.event_name == 'pull_request' &&
        github.event.pull_request.head.repo.fork == false &&
        github.event.pull_request.user.login != 'dependabot[bot]'
      }}
    permissions:
      pull-requests: write
      contents: read
    steps:
      - name: Checkout repository
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Download all playwright reports
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          pattern: playwright-report-*
          path: reports

      - name: Deploy reports and generate section
        env:
          CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
          CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
          GITHUB_TOKEN: ${{ github.token }}
          GITHUB_SHA: ${{ github.event.pull_request.head.sha }}
          SUMMARY_FILE: playwright-section.md
          BRANCH_NAME: ${{ github.head_ref }}
        run: |
          bash ./scripts/cicd/pr-playwright-deploy-and-comment.sh \
            "${{ github.event.pull_request.number }}" \
            "$BRANCH_NAME" \
            "completed"

      - name: Read and upsert playwright section
        if: ${{ !cancelled() }}
        uses: ./.github/actions/read-and-upsert-comment-section
        with:
          pr-number: ${{ github.event.pull_request.number }}
          section-name: playwright
          section-file: playwright-section.md
          token: ${{ github.token }}