regression-tests.yml

name: Regression Tests

on:
  deployment_status:
    states: ["success"]

env:
  VERCEL_PROTECTION_BYPASS: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }}

jobs:
  detect-provider:
    name: Detect Deployment Provider
    runs-on: ubuntu-latest
    outputs:
      provider: ${{ steps.detect.outputs.provider }}
      production-url: ${{ steps.detect.outputs.production-url }}
      branch-label: ${{ steps.detect.outputs.branch-label }}
      is-preview: ${{ steps.detect.outputs.is-preview }}
    steps:
      - uses: actions/checkout@v4

      - name: Detect provider and production URL
        id: detect
        run: |
          CREATOR="${{ github.event.deployment_status.creator.login }}"
          ENVIRONMENT="${{ github.event.deployment.environment }}"

          if [[ "$ENVIRONMENT" == "Preview" ]]; then
            echo "is-preview=true" >> $GITHUB_OUTPUT
          else
            echo "is-preview=false" >> $GITHUB_OUTPUT
          fi

          if [[ "$CREATOR" == "vercel[bot]" ]]; then
            echo "provider=vercel" >> $GITHUB_OUTPUT
            PKG_NAME=$(node -p "require('./core/package.json').name")

            if [[ "$ENVIRONMENT" == "Preview" ]]; then
              case "$PKG_NAME" in
                "@bigcommerce/catalyst-core")
                  echo "production-url=https://canary.catalyst-demo.site/" >> $GITHUB_OUTPUT ;;
                "@bigcommerce/catalyst-makeswift")
                  echo "production-url=https://canary.makeswift.catalyst-demo.site" >> $GITHUB_OUTPUT ;;
                *)
                  echo "::warning::No production URL configured for package: $PKG_NAME. Skipping comparison."
                  echo "production-url=" >> $GITHUB_OUTPUT ;;
              esac
              echo "branch-label=" >> $GITHUB_OUTPUT
            else
              case "$PKG_NAME" in
                "@bigcommerce/catalyst-core")
                  echo "branch-label=canary" >> $GITHUB_OUTPUT
                  echo "production-url=${{ github.event.deployment_status.target_url }}" >> $GITHUB_OUTPUT ;;
                "@bigcommerce/catalyst-makeswift")
                  echo "branch-label=integrations/makeswift" >> $GITHUB_OUTPUT
                  echo "production-url=${{ github.event.deployment_status.target_url }}" >> $GITHUB_OUTPUT ;;
                *)
                  echo "branch-label=" >> $GITHUB_OUTPUT
                  echo "production-url=" >> $GITHUB_OUTPUT ;;
              esac
            fi

          elif [[ "$CREATOR" == "cloudflare-pages[bot]" ]]; then
            echo "provider=cloudflare" >> $GITHUB_OUTPUT
            echo "::warning::Cloudflare production URL not yet configured. Skipping comparison."
            echo "production-url=" >> $GITHUB_OUTPUT
            echo "branch-label=" >> $GITHUB_OUTPUT

          else
            echo "::warning::Unknown deployment provider: $CREATOR. Skipping audits."
            echo "provider=unknown" >> $GITHUB_OUTPUT
            echo "production-url=" >> $GITHUB_OUTPUT
            echo "branch-label=" >> $GITHUB_OUTPUT
          fi

  unlighthouse-audit-preview:
    name: Unlighthouse Audit Preview (${{ needs.detect-provider.outputs.provider }}) - ${{ matrix.device }}
    needs: [detect-provider]
    if: needs.detect-provider.outputs.is-preview == 'true'
    runs-on: ubuntu-latest
    concurrency:
      group: regression-preview-${{ github.event.deployment.ref }}-${{ matrix.device }}
      cancel-in-progress: true
    strategy:
      matrix:
        device: [desktop, mobile]

    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Install Dependencies
        run: npm install @unlighthouse/cli puppeteer -g

      - name: Unlighthouse audit on ${{ matrix.device }} (preview)
        env:
          PROVIDER: ${{ needs.detect-provider.outputs.provider }}
          PREVIEW_URL: ${{ github.event.deployment_status.target_url }}
        run: |
          if [[ "$PROVIDER" == "vercel" ]]; then
            unlighthouse-ci --site "$PREVIEW_URL" --${{ matrix.device }} --disable-robots-txt \
              --extra-headers "x-vercel-protection-bypass=$VERCEL_PROTECTION_BYPASS,x-vercel-set-bypass-cookie=true"
          else
            unlighthouse-ci --site "$PREVIEW_URL" --${{ matrix.device }} --disable-robots-txt
          fi

      - name: Upload ${{ matrix.device }} preview audit
        if: failure() || success()
        uses: actions/upload-artifact@v4
        with:
          name: unlighthouse-preview-${{ matrix.device }}-report
          path: "./.unlighthouse/"
          include-hidden-files: "true"

  unlighthouse-audit-production:
    name: Unlighthouse Audit Production (${{ needs.detect-provider.outputs.provider }}) - ${{ matrix.device }}
    needs: [detect-provider]
    if: needs.detect-provider.outputs.production-url != ''
    runs-on: ubuntu-latest
    concurrency:
      group: regression-production-${{ github.event.deployment.environment == 'Preview' && github.event.deployment.ref || github.event.deployment.sha }}-${{ matrix.device }}
      cancel-in-progress: ${{ github.event.deployment.environment == 'Preview' }}
    strategy:
      matrix:
        device: [desktop, mobile]

    steps:
      - uses: actions/checkout@v4

      - name: Install Dependencies
        run: npm install @unlighthouse/cli puppeteer -g

      - name: Unlighthouse audit on ${{ matrix.device }} (production)
        env:
          PRODUCTION_URL: ${{ needs.detect-provider.outputs.production-url }}
        run: unlighthouse-ci --site "$PRODUCTION_URL" --${{ matrix.device }} --disable-robots-txt

      - name: Upload ${{ matrix.device }} production audit
        if: failure() || success()
        uses: actions/upload-artifact@v4
        with:
          name: unlighthouse-production-${{ matrix.device }}-report
          path: "./.unlighthouse/"
          include-hidden-files: "true"

  unlighthouse-compare:
    name: Unlighthouse Compare & Comment (${{ needs.detect-provider.outputs.provider }})
    needs: [detect-provider, unlighthouse-audit-preview, unlighthouse-audit-production]
    if: needs.detect-provider.outputs.is-preview == 'true' && needs.detect-provider.outputs.production-url != ''
    runs-on: ubuntu-latest
    permissions:
      pull-requests: write

    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-node@v4
        with:
          node-version-file: ".nvmrc"

      - name: Download all Unlighthouse artifacts
        uses: actions/download-artifact@v4
        with:
          pattern: unlighthouse-*-report
          path: /tmp/unlighthouse-artifacts
          merge-multiple: false

      - name: Compare audits
        env:
          PROVIDER: ${{ needs.detect-provider.outputs.provider }}
        run: |
          node .github/scripts/compare-unlighthouse.mts \
            --preview-desktop    /tmp/unlighthouse-artifacts/unlighthouse-preview-desktop-report/ci-result.json \
            --preview-mobile     /tmp/unlighthouse-artifacts/unlighthouse-preview-mobile-report/ci-result.json \
            --production-desktop /tmp/unlighthouse-artifacts/unlighthouse-production-desktop-report/ci-result.json \
            --production-mobile  /tmp/unlighthouse-artifacts/unlighthouse-production-mobile-report/ci-result.json \
            --output /tmp/unlighthouse-report.md \
            --meta-output /tmp/unlighthouse-meta.json \
            --provider "$PROVIDER"
          cat /tmp/unlighthouse-report.md >> "$GITHUB_STEP_SUMMARY"

      - name: Post PR comment
        uses: actions/github-script@v7
        with:
          script: |
            const postComment = require('./.github/scripts/post-unlighthouse-pr-comment.js')
            await postComment({
              github,
              context,
              provider: '${{ needs.detect-provider.outputs.provider }}',
              reportPath: '/tmp/unlighthouse-report.md',
              metaPath: '/tmp/unlighthouse-meta.json',
            })

  unlighthouse-report:
    name: Unlighthouse Report (${{ needs.detect-provider.outputs.provider }}) — ${{ needs.detect-provider.outputs.branch-label }}
    needs: [detect-provider, unlighthouse-audit-production]
    if: needs.detect-provider.outputs.is-preview == 'false' && needs.detect-provider.outputs.production-url != ''
    runs-on: ubuntu-latest
    permissions:
      contents: write

    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-node@v4
        with:
          node-version-file: ".nvmrc"

      - name: Download production Unlighthouse artifacts
        uses: actions/download-artifact@v4
        with:
          pattern: unlighthouse-production-*-report
          path: /tmp/unlighthouse-artifacts
          merge-multiple: false

      - name: Format report
        run: |
          node .github/scripts/audit-unlighthouse.mts \
            --desktop /tmp/unlighthouse-artifacts/unlighthouse-production-desktop-report/ci-result.json \
            --mobile  /tmp/unlighthouse-artifacts/unlighthouse-production-mobile-report/ci-result.json \
            --branch  "${{ needs.detect-provider.outputs.branch-label }}" \
            --output  /tmp/unlighthouse-report.md
          cat /tmp/unlighthouse-report.md >> "$GITHUB_STEP_SUMMARY"

      - name: Post commit comment
        uses: actions/github-script@v7
        with:
          script: |
            const postReport = require('./.github/scripts/post-unlighthouse-commit-comment.js')
            await postReport({ github, context, reportPath: '/tmp/unlighthouse-report.md' })