diff --git a/.github/workflows/flaky-e2e-check.yml b/.github/workflows/flaky-e2e-check.yml new file mode 100644 index 00000000000000..93fbacc1875d2b --- /dev/null +++ b/.github/workflows/flaky-e2e-check.yml @@ -0,0 +1,175 @@ +name: Flaky E2E Check +on: + pull_request: + paths: + - '**/*.e2e.ts' + +permissions: + actions: write + contents: read + +env: + NODE_OPTIONS: --max-old-space-size=4096 + ALLOWED_HOSTNAMES: ${{ vars.CI_ALLOWED_HOSTNAMES }} + CALENDSO_ENCRYPTION_KEY: ${{ secrets.CI_CALENDSO_ENCRYPTION_KEY }} + DAILY_API_KEY: ${{ secrets.CI_DAILY_API_KEY }} + DATABASE_URL: ${{ secrets.CI_DATABASE_URL }} + DATABASE_DIRECT_URL: ${{ secrets.CI_DATABASE_URL }} + DEPLOYSENTINEL_API_KEY: ${{ secrets.DEPLOYSENTINEL_API_KEY }} + E2E_TEST_APPLE_CALENDAR_EMAIL: ${{ secrets.E2E_TEST_APPLE_CALENDAR_EMAIL }} + E2E_TEST_APPLE_CALENDAR_PASSWORD: ${{ secrets.E2E_TEST_APPLE_CALENDAR_PASSWORD }} + E2E_TEST_MAILHOG_ENABLED: ${{ vars.E2E_TEST_MAILHOG_ENABLED }} + GOOGLE_API_CREDENTIALS: ${{ secrets.CI_GOOGLE_API_CREDENTIALS }} + EMAIL_SERVER_HOST: ${{ secrets.CI_EMAIL_SERVER_HOST }} + EMAIL_SERVER_PORT: ${{ secrets.CI_EMAIL_SERVER_PORT }} + EMAIL_SERVER_USER: ${{ secrets.CI_EMAIL_SERVER_USER }} + EMAIL_SERVER_PASSWORD: ${{ secrets.CI_EMAIL_SERVER_PASSWORD}} + GOOGLE_LOGIN_ENABLED: ${{ vars.CI_GOOGLE_LOGIN_ENABLED }} + NEXTAUTH_SECRET: ${{ secrets.CI_NEXTAUTH_SECRET }} + NEXTAUTH_URL: ${{ secrets.CI_NEXTAUTH_URL }} + NEXT_PUBLIC_API_V2_URL: ${{ secrets.CI_NEXT_PUBLIC_API_V2_URL }} + NEXT_PUBLIC_API_V2_ROOT_URL: ${{ secrets.CI_NEXT_PUBLIC_API_V2_ROOT_URL }} + NEXT_PUBLIC_IS_E2E: ${{ vars.CI_NEXT_PUBLIC_IS_E2E }} + NEXT_PUBLIC_ORG_SELF_SERVE_ENABLED: ${{ vars.CI_NEXT_PUBLIC_ORG_SELF_SERVE_ENABLED }} + NEXT_PUBLIC_STRIPE_PUBLIC_KEY: ${{ secrets.CI_NEXT_PUBLIC_STRIPE_PUBLIC_KEY }} + NEXT_PUBLIC_WEBAPP_URL: ${{ vars.CI_NEXT_PUBLIC_WEBAPP_URL }} + NEXT_PUBLIC_WEBSITE_URL: ${{ vars.CI_NEXT_PUBLIC_WEBSITE_URL }} + PAYMENT_FEE_FIXED: ${{ vars.CI_PAYMENT_FEE_FIXED }} + PAYMENT_FEE_PERCENTAGE: ${{ vars.CI_PAYMENT_FEE_PERCENTAGE }} + SAML_ADMINS: ${{ secrets.CI_SAML_ADMINS }} + SAML_DATABASE_URL: ${{ secrets.CI_SAML_DATABASE_URL }} + STRIPE_PRIVATE_KEY: ${{ secrets.CI_STRIPE_PRIVATE_KEY }} + STRIPE_CLIENT_ID: ${{ secrets.CI_STRIPE_CLIENT_ID }} + STRIPE_WEBHOOK_SECRET: ${{ secrets.CI_STRIPE_WEBHOOK_SECRET }} + SENDGRID_API_KEY: ${{ secrets.CI_SENDGRID_API_KEY }} + SENDGRID_EMAIL: ${{ secrets.CI_SENDGRID_EMAIL }} + TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} + TURBO_TEAM: ${{ secrets.TURBO_TEAM }} + +jobs: + detect-changed-tests: + runs-on: ubuntu-latest + outputs: + changed_tests: ${{ steps.filter.outputs.tests }} + steps: + - uses: actions/checkout@v4 + + - name: Get Changed Files + id: filter + run: | + CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }} -- '**/*.e2e.ts') + echo "Changed Test Files:" + echo "$CHANGED_FILES" + + if [[ -z "$CHANGED_FILES" ]]; then + echo "tests=[]" >> "$GITHUB_OUTPUT" + else + JSON_LIST=$(echo "$CHANGED_FILES" | xargs -n1 basename | jq -R -s -c 'split("\n")[:-1]') + echo "tests=$JSON_LIST" >> "$GITHUB_OUTPUT" + fi + + flaky-check: + needs: detect-changed-tests + if: needs.detect-changed-tests.outputs.changed_tests != '[]' + timeout-minutes: 30 + runs-on: buildjet-8vcpu-ubuntu-2204 + strategy: + fail-fast: false + matrix: + iteration: [1, 2, 3, 4, 5] + + services: + postgres: + image: postgres:13 + credentials: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: calendso + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + mailhog: + image: mailhog/mailhog:v1.0.1 + credentials: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + ports: + - 8025:8025 + - 1025:1025 + + mailhog: + image: mailhog/mailhog:v1.0.1 + ports: + - 8025:8025 + - 1025:1025 + + steps: + - uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - uses: actions/checkout@v4 + - uses: ./.github/actions/yarn-install + - uses: ./.github/actions/yarn-playwright-install + - uses: ./.github/actions/cache-db + - uses: ./.github/actions/cache-build + - name: Run Flaky Test Check (Iteration ${{ matrix.iteration }}) + run: | + CHANGED_TESTS=${{ needs.detect-changed-tests.outputs.changed_tests }} + for test in $(echo "$CHANGED_TESTS" | jq -r '.[]'); do + echo "Running: $test (Iteration ${{ matrix.iteration }})" + yarn playwright test "$test" && echo "$test: PASS" >> flaky-results.txt || echo "$test: FAIL" >> flaky-results.txt + done + + - name: Calculate Flakiness + if: matrix.iteration == 5 + run: | + echo "Calculating flakiness..." + ITERATIONS=5 + THRESHOLD=40 # Flakiness % to consider test as flaky + FLAKY_TESTS_FOUND=false + + echo -e "Test Report:\n" > flaky-report.txt + + while read -r test; do + test_name=$(echo "$test" | cut -d':' -f1) + passes=$(grep -c "$test_name: PASS" flaky-results.txt) + failures=$((ITERATIONS - passes)) + flakiness=$(( (failures * 100) / ITERATIONS )) + + # Determine status + if [ "$flakiness" -eq 0 ]; then + status="✅ Stable" + elif [ "$flakiness" -eq 100 ]; then + status="❌ Always Failing" + elif [ "$flakiness" -ge "$THRESHOLD" ]; then + status="⚠️ Flaky" + FLAKY_TESTS_FOUND=true + else + status="✅ Mostly Stable" + fi + + echo "$test_name - $status ($flakiness% failure rate)" >> flaky-report.txt + done < <(awk -F': ' '{print $1}' flaky-results.txt | sort -u) + + cat flaky-report.txt + + if [ "$FLAKY_TESTS_FOUND" = true ]; then + echo "❗ Flaky tests detected! Check flaky-report.txt." + exit 1 + fi + + - name: Upload Flakiness Report + uses: actions/upload-artifact@v4 + if: always() + with: + name: flaky-report + path: flaky-report.txt + retention-days: 30 diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index d1b72c53d172c5..2be7cb51b486da 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -195,6 +195,13 @@ jobs: uses: ./.github/workflows/e2e-embed-react.yml secrets: inherit + flaky-e2e-check: + name: Flaky E2E Check + needs: [changes, check-label, build, build-api-v1, build-api-v2] + if: ${{ needs.check-label.outputs.run-e2e == 'true' && needs.changes.outputs.has-files-requiring-all-checks == 'true' }} + uses: ./.github/workflows/flaky-e2e-check.yml + secrets: inherit + analyze: name: Analyze Build needs: [build]