chore: workflow to check for flaky e2e changes

.github/workflows/flaky-e2e-check.yml

@@ -0,0 +1,175 @@
+name: Flaky E2E Check
+on:
+  pull_request:
+    paths:
+      - '**/*.e2e.ts'
+
+permissions:
+  actions: write
+  contents: read
+
+env:
+  NODE_OPTIONS: --max-old-space-size=4096
+  ALLOWED_HOSTNAMES: ${{ vars.CI_ALLOWED_HOSTNAMES }}
+  CALENDSO_ENCRYPTION_KEY: ${{ secrets.CI_CALENDSO_ENCRYPTION_KEY }}
+  DAILY_API_KEY: ${{ secrets.CI_DAILY_API_KEY }}
+  DATABASE_URL: ${{ secrets.CI_DATABASE_URL }}
+  DATABASE_DIRECT_URL: ${{ secrets.CI_DATABASE_URL }}
+  DEPLOYSENTINEL_API_KEY: ${{ secrets.DEPLOYSENTINEL_API_KEY }}
+  E2E_TEST_APPLE_CALENDAR_EMAIL: ${{ secrets.E2E_TEST_APPLE_CALENDAR_EMAIL }}
+  E2E_TEST_APPLE_CALENDAR_PASSWORD: ${{ secrets.E2E_TEST_APPLE_CALENDAR_PASSWORD }}
+  E2E_TEST_MAILHOG_ENABLED: ${{ vars.E2E_TEST_MAILHOG_ENABLED }}
+  GOOGLE_API_CREDENTIALS: ${{ secrets.CI_GOOGLE_API_CREDENTIALS }}
+  EMAIL_SERVER_HOST: ${{ secrets.CI_EMAIL_SERVER_HOST }}
+  EMAIL_SERVER_PORT: ${{ secrets.CI_EMAIL_SERVER_PORT }}
+  EMAIL_SERVER_USER: ${{ secrets.CI_EMAIL_SERVER_USER }}
+  EMAIL_SERVER_PASSWORD: ${{ secrets.CI_EMAIL_SERVER_PASSWORD}}
+  GOOGLE_LOGIN_ENABLED: ${{ vars.CI_GOOGLE_LOGIN_ENABLED }}
+  NEXTAUTH_SECRET: ${{ secrets.CI_NEXTAUTH_SECRET }}
+  NEXTAUTH_URL: ${{ secrets.CI_NEXTAUTH_URL }}
+  NEXT_PUBLIC_API_V2_URL: ${{ secrets.CI_NEXT_PUBLIC_API_V2_URL }}
+  NEXT_PUBLIC_API_V2_ROOT_URL: ${{ secrets.CI_NEXT_PUBLIC_API_V2_ROOT_URL }}
+  NEXT_PUBLIC_IS_E2E: ${{ vars.CI_NEXT_PUBLIC_IS_E2E }}
+  NEXT_PUBLIC_ORG_SELF_SERVE_ENABLED: ${{ vars.CI_NEXT_PUBLIC_ORG_SELF_SERVE_ENABLED }}
+  NEXT_PUBLIC_STRIPE_PUBLIC_KEY: ${{ secrets.CI_NEXT_PUBLIC_STRIPE_PUBLIC_KEY }}
+  NEXT_PUBLIC_WEBAPP_URL: ${{ vars.CI_NEXT_PUBLIC_WEBAPP_URL }}
+  NEXT_PUBLIC_WEBSITE_URL: ${{ vars.CI_NEXT_PUBLIC_WEBSITE_URL }}
+  PAYMENT_FEE_FIXED: ${{ vars.CI_PAYMENT_FEE_FIXED }}
+  PAYMENT_FEE_PERCENTAGE: ${{ vars.CI_PAYMENT_FEE_PERCENTAGE }}
+  SAML_ADMINS: ${{ secrets.CI_SAML_ADMINS }}
+  SAML_DATABASE_URL: ${{ secrets.CI_SAML_DATABASE_URL }}
+  STRIPE_PRIVATE_KEY: ${{ secrets.CI_STRIPE_PRIVATE_KEY }}
+  STRIPE_CLIENT_ID: ${{ secrets.CI_STRIPE_CLIENT_ID }}
+  STRIPE_WEBHOOK_SECRET: ${{ secrets.CI_STRIPE_WEBHOOK_SECRET }}
+  SENDGRID_API_KEY: ${{ secrets.CI_SENDGRID_API_KEY }}
+  SENDGRID_EMAIL: ${{ secrets.CI_SENDGRID_EMAIL }}
+  TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
+  TURBO_TEAM: ${{ secrets.TURBO_TEAM }}
+
+jobs:
+  detect-changed-tests:
+    runs-on: ubuntu-latest
+    outputs:
+      changed_tests: ${{ steps.filter.outputs.tests }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Get Changed Files
+        id: filter
+        run: |
+          CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }} -- '**/*.e2e.ts')
+          echo "Changed Test Files:"
+          echo "$CHANGED_FILES"
+
+          if [[ -z "$CHANGED_FILES" ]]; then
+            echo "tests=[]" >> "$GITHUB_OUTPUT"
+          else
+            JSON_LIST=$(echo "$CHANGED_FILES" | xargs -n1 basename | jq -R -s -c 'split("\n")[:-1]')
+            echo "tests=$JSON_LIST" >> "$GITHUB_OUTPUT"
+          fi
+
+  flaky-check:
+    needs: detect-changed-tests
+    if: needs.detect-changed-tests.outputs.changed_tests != '[]'
+    timeout-minutes: 30
+    runs-on: buildjet-8vcpu-ubuntu-2204
+    strategy:
+      fail-fast: false
+      matrix:
+        iteration: [1, 2, 3, 4, 5]
+
+    services:
+      postgres:
+        image: postgres:13
+        credentials:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: calendso
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+      mailhog:
+        image: mailhog/mailhog:v1.0.1
+        credentials:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+        ports:
+          - 8025:8025
+          - 1025:1025
+
+      mailhog:
+        image: mailhog/mailhog:v1.0.1
+        ports:
+          - 8025:8025
+          - 1025:1025
+
+    steps:
+      - uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/yarn-install
+      - uses: ./.github/actions/yarn-playwright-install
+      - uses: ./.github/actions/cache-db
+      - uses: ./.github/actions/cache-build
+      - name: Run Flaky Test Check (Iteration ${{ matrix.iteration }})
+        run: |
+          CHANGED_TESTS=${{ needs.detect-changed-tests.outputs.changed_tests }}
+          for test in $(echo "$CHANGED_TESTS" | jq -r '.[]'); do
+            echo "Running: $test (Iteration ${{ matrix.iteration }})"
+            yarn playwright test "$test" && echo "$test: PASS" >> flaky-results.txt || echo "$test: FAIL" >> flaky-results.txt
+          done
+
+      - name: Calculate Flakiness
+        if: matrix.iteration == 5
+        run: |
+          echo "Calculating flakiness..."
+          ITERATIONS=5
+          THRESHOLD=40  # Flakiness % to consider test as flaky
+          FLAKY_TESTS_FOUND=false
+
+          echo -e "Test Report:\n" > flaky-report.txt
+
+          while read -r test; do
+            test_name=$(echo "$test" | cut -d':' -f1)
+            passes=$(grep -c "$test_name: PASS" flaky-results.txt)
+            failures=$((ITERATIONS - passes))
+            flakiness=$(( (failures * 100) / ITERATIONS ))
+
+            # Determine status
+            if [ "$flakiness" -eq 0 ]; then
+              status="✅ Stable"
+            elif [ "$flakiness" -eq 100 ]; then
+              status="❌ Always Failing"
+            elif [ "$flakiness" -ge "$THRESHOLD" ]; then
+              status="⚠️ Flaky"
+              FLAKY_TESTS_FOUND=true
+            else
+              status="✅ Mostly Stable"
+            fi
+
+            echo "$test_name - $status ($flakiness% failure rate)" >> flaky-report.txt
+          done < <(awk -F': ' '{print $1}' flaky-results.txt | sort -u)
+
+          cat flaky-report.txt
+
+          if [ "$FLAKY_TESTS_FOUND" = true ]; then
+            echo "❗ Flaky tests detected! Check flaky-report.txt."
+            exit 1
+          fi
+
+      - name: Upload Flakiness Report
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: flaky-report
+          path: flaky-report.txt
+          retention-days: 30

.github/workflows/pr.yml

@@ -195,6 +195,13 @@ jobs:
     uses: ./.github/workflows/e2e-embed-react.yml
     secrets: inherit
 
+  flaky-e2e-check:
+    name: Flaky E2E Check
+    needs: [changes, check-label, build, build-api-v1, build-api-v2]
+    if: ${{ needs.check-label.outputs.run-e2e == 'true' && needs.changes.outputs.has-files-requiring-all-checks == 'true' }}
+    uses: ./.github/workflows/flaky-e2e-check.yml
+    secrets: inherit
+
   analyze:
     name: Analyze Build
     needs: [build]