Update E2E NPM scripts (#10276)

Co-authored-by: Allie Mims <[email protected]>
Co-authored-by: Allie Mims <[email protected]>

.github/actions/e2e-pw/run-log-tests/action.yml

@@ -8,7 +8,7 @@ runs:
           id: first_run_e2e_pw_tests
           # Use +e to trap errors when running E2E tests.
           shell: /bin/bash +e {0}
-          run: npm run test:e2e-pw-ci
+          run: npm run test:e2e-ci
         - uses: actions/upload-artifact@v4
           if: always()
           with:

.github/actions/e2e/atomic-prepare-and-run/action.yml

@@ -25,5 +25,9 @@ runs:
       run: |
         mkdir screenshots
 
+    - name: Install Playwright
+      shell: bash
+      run: npx playwright install chromium
+
     - name: "Run tests, upload screenshots & logs"
       uses: ./.github/actions/e2e/run-log-tests

.github/actions/e2e/run-log-tests/action.yml

@@ -9,19 +9,18 @@ runs:
       # Use +e to trap errors when running E2E tests.
       shell: /bin/bash +e {0}
       run: |
-        npm run test:e2e -- --json --outputFile="$E2E_RESULT_FILEPATH" --NODE_ENV="$NODE_ENV"
+        npm run test:e2e-ci
         
         if [[ -f "$E2E_RESULT_FILEPATH" ]]; then
-          E2E_NUM_FAILED_TEST_SUITES=$(cat "$E2E_RESULT_FILEPATH" | jq '.numFailedTestSuites')
+          E2E_NUM_FAILED_TEST_SUITES=$(cat "$E2E_RESULT_FILEPATH" | jq '.stats["unexpected"]')
           echo "FIRST_RUN_FAILED_TEST_SUITES=$(echo $E2E_NUM_FAILED_TEST_SUITES)" >> $GITHUB_OUTPUT
           if [[ ${E2E_NUM_FAILED_TEST_SUITES} -gt 0 ]]; then
             echo "::notice::${E2E_NUM_FAILED_TEST_SUITES} test suite(s) failed in the first run but we will try (it) them again in the second run."
             exit 0
           fi
         else
           echo "FIRST_RUN_FAILED_TEST_SUITES=0" >> $GITHUB_OUTPUT
-          echo "::notice::${E2E_RESULT_FILEPATH} doesn't exist so results are unclear and we will not try again."
-          exit 1
+          exit 0
         fi
 
     # Retry failed E2E tests
@@ -30,18 +29,17 @@ runs:
       shell: bash
       # Filter failed E2E files from the result JSON file, and re-run them.
       run: |
-        cat "$E2E_RESULT_FILEPATH" | jq '.testResults[] | select(.status == "failed") | .name' | xargs npm run test:e2e -- --NODE_ENV="$NODE_ENV"
+        npm run test:e2e-ci $(cat $E2E_RESULT_FILEPATH | jq -r '[.suites[] | (if has("suites") then .suites[] | .specs[] else .specs[] end) | select(.tests[].status == "unexpected") | .file] | unique | .[]')
 
     # Archive screenshots if any
     - name: Archive e2e test screenshots & logs
-      if: ${{ failure() }}
+      if: ${{ always() }}
       uses: actions/upload-artifact@v4
       with:
           name: wp(${{ env.E2E_WP_VERSION }})-wc(${{ env.E2E_WC_VERSION }})-${{ env.E2E_GROUP }}-${{ env.E2E_BRANCH }}
           path: |
-            screenshots
-            tests/e2e/screenshots
-            tests/e2e/docker/wordpress/wp-content/debug.log
+            playwright-report/
+            tests/e2e-pw/test-results
             ${{ env.E2E_RESULT_FILEPATH }}
           if-no-files-found: ignore
           retention-days: 14

.github/workflows/e2e-pull-request.yml

@@ -3,9 +3,8 @@ name: E2E Tests - Pull Request
 on:
   pull_request:
     branches:
-      # - develop
-      # - trunk
-      - dev/10240-update-npm-scripts # [TODO] Remove this line after the branch is merged.
+      - develop
+      - trunk
   workflow_dispatch:
   workflow_call:
     inputs:
@@ -31,7 +30,7 @@ env:
   E2E_SLACK_CHANNEL:             ${{ secrets.E2E_SLACK_CHANNEL }}
   E2E_SLACK_TOKEN:               ${{ secrets.E2E_SLACK_TOKEN }}
   E2E_USE_LOCAL_SERVER:          false
-  E2E_RESULT_FILEPATH:           'tests/e2e/results.json'
+  E2E_RESULT_FILEPATH:           'tests/e2e-pw/results.json'
   WCPAY_USE_BUILD_ARTIFACT:      ${{ inputs.wcpay-use-build-artifact }}
   WCPAY_ARTIFACT_DIRECTORY:      'zipfile'
   NODE_ENV:                      'test'
@@ -75,5 +74,9 @@ jobs:
       - name: Setup E2E environment
         uses: ./.github/actions/e2e/env-setup
 
+      - name: Install Playwright
+        shell: bash
+        run: npx playwright install chromium
+
       - name: Run tests, upload screenshots & logs
         uses: ./.github/actions/e2e/run-log-tests

.github/workflows/e2e-test.yml

@@ -22,7 +22,7 @@ env:
   E2E_SLACK_CHANNEL:             ${{ secrets.E2E_SLACK_CHANNEL }}
   E2E_SLACK_TOKEN:               ${{ secrets.E2E_SLACK_TOKEN }}
   E2E_USE_LOCAL_SERVER:          false
-  E2E_RESULT_FILEPATH:           'tests/e2e/results.json'
+  E2E_RESULT_FILEPATH:           'tests/e2e-pw/results.json'
   WC_MIN_SUPPORTED_VERSION:      '7.6.0'
   NODE_ENV:                      'test'
   FORCE_E2E_DEPS_SETUP:          true
@@ -64,6 +64,10 @@ jobs:
       - name: Setup E2E environment
         uses: ./.github/actions/e2e/env-setup
 
+      - name: Install Playwright
+        shell: bash
+        run: npx playwright install chromium
+
       - name: Run tests, upload screenshots & logs
         uses: ./.github/actions/e2e/run-log-tests
 
@@ -116,5 +120,9 @@ jobs:
       - name: Setup E2E environment
         uses: ./.github/actions/e2e/env-setup
 
+      - name: Install Playwright
+        shell: bash
+        run: npx playwright install chromium
+
       - name: Run tests, upload screenshots & logs
         uses: ./.github/actions/e2e/run-log-tests

.github/workflows/e2e-tests-atomic.yml

@@ -10,7 +10,7 @@ env:
   E2E_SLACK_TOKEN: ${{ secrets.E2E_SLACK_TOKEN }}
   E2E_WP_VERSION: 'nightly'
   E2E_WC_VERSION: 'latest'
-  E2E_RESULT_FILEPATH: 'tests/e2e/results.json'
+  E2E_RESULT_FILEPATH: 'tests/e2e-pw/results.json'
   NODE_ENV: 'atomic'
 
 concurrency:

changelog/dev-10238-update-e2e-tests-all-workflow

@@ -0,0 +1,4 @@
+Significance: patch
+Type: dev
+
+Update Github actions and workflows. Change e2e tests directory structure.

changelog/dev-10240-update-npm-scripts

@@ -0,0 +1,4 @@
+Significance: patch
+Type: dev
+
+Update E2E NPM scripts.

package.json

@@ -34,13 +34,11 @@
     "test:e2e-up": "./tests/e2e/env/up.sh",
     "test:e2e-cleanup": "./tests/e2e/env/cleanup.sh",
     "test:e2e-reset": "npm run test:e2e-down && npm run test:e2e-cleanup",
-    "test:e2e": "NODE_CONFIG_DIR='tests/e2e/config' JEST_PUPPETEER_CONFIG='tests/e2e/config/jest-puppeteer-headless.config.js' wp-scripts test-e2e --config tests/e2e/config/jest.config.js",
-    "test:e2e-dev": "NODE_CONFIG_DIR='tests/e2e/config' JEST_PUPPETEER_CONFIG='tests/e2e/config/jest-puppeteer.config.js' wp-scripts test-e2e --config tests/e2e/config/jest.config.js --puppeteer-interactive",
+    "test:e2e": "./tests/e2e-pw/test-e2e-pw.sh",
+    "test:e2e-ci": "npx playwright test --config=tests/e2e-pw/playwright.config.ts --grep-invert @todo",
+    "test:e2e-ui": "./tests/e2e-pw/test-e2e-pw-ui.sh",
     "test:e2e-performance": "NODE_CONFIG_DIR='tests/e2e/config' wp-scripts test-e2e --config tests/e2e/config/jest.performance.config.js",
-    "test:e2e-pw": "./tests/e2e-pw/test-e2e-pw.sh",
-    "test:e2e-pw-update-snapshots": "npm run test:e2e-pw -- --update-snapshots",
-    "test:e2e-pw-ui": "./tests/e2e-pw/test-e2e-pw-ui.sh",
-    "test:e2e-pw-ci": "npx playwright test --config=tests/e2e-pw/playwright.config.ts --grep-invert @todo",
+    "test:e2e-update-snapshots": "npm run test:e2e -- --update-snapshots",
     "test:update-snapshots": "npm run test:js -- --updateSnapshot",
     "test:php": "./bin/run-tests.sh",
     "test:php-coverage": "./bin/check-test-coverage.sh",
@@ -96,7 +94,7 @@
   "devDependencies": {
     "@automattic/color-studio": "4.0.0",
     "@jest/test-sequencer": "29.5.0",
-    "@playwright/test": "1.43.1",
+    "@playwright/test": "1.50.1",
     "@pmmmwh/react-refresh-webpack-plugin": "0.5.7",
     "@testing-library/jest-dom": "5.14.1",
     "@testing-library/react": "11.2.5",

tests/e2e-pw/config/default.ts

@@ -20,16 +20,47 @@ export const config = {
 		},
 	},
 	products: {
+		cap: {
+			name: 'Cap',
+			pageNumber: 1,
+		},
+		belt: {
+			name: 'Belt',
+			pageNumber: 1,
+		},
 		simple: {
 			name: 'Beanie',
+			pageNumber: 1,
+		},
+		sunglasses: {
+			name: 'Sunglasses',
+			pageNumber: 2,
 		},
 		variable: {
 			name: 'Variable Product with Three Variations',
+			pageNumber: 1,
 		},
 		grouped: {
 			name: 'Grouped Product with Three Children',
+			pageNumber: 1,
 		},
-	},
+		hoodie_with_logo: {
+			name: 'Hoodie with Logo',
+			pageNumber: 1,
+		},
+		subscription_signup_fee: {
+			name: 'Subscription signup fee product',
+			pageNumber: 2,
+		},
+		subscription_no_signup_fee: {
+			name: 'Subscription no signup fee product',
+			pageNumber: 2,
+		},
+		subscription_free_trial: {
+			name: 'Subscription free trial product',
+			pageNumber: 2,
+		},
+	} as Record< string, Product >,
 	addresses: {
 		admin: {
 			store: {
@@ -314,3 +345,5 @@ export type CustomerAddress = Omit<
 > & {
 	state?: string;
 };
+
+export type Product = { name: string; pageNumber: number };