Update e2e tests all and atomic workflows (#10277)

Co-authored-by: Eduardo Umpierre <[email protected]>

.github/actions/e2e/run-log-tests/action.yml

@@ -9,10 +9,10 @@ runs:
       # Use +e to trap errors when running E2E tests.
       shell: /bin/bash +e {0}
       run: |
-        npm run test:e2e -- --json --outputFile="$E2E_RESULT_FILEPATH" --NODE_ENV="$NODE_ENV"
+        npm run test:e2e-ci
         
         if [[ -f "$E2E_RESULT_FILEPATH" ]]; then
-          E2E_NUM_FAILED_TEST_SUITES=$(cat "$E2E_RESULT_FILEPATH" | jq '.numFailedTestSuites')
+          E2E_NUM_FAILED_TEST_SUITES=$(cat "$E2E_RESULT_FILEPATH" | jq '.stats["unexpected"]')
           echo "FIRST_RUN_FAILED_TEST_SUITES=$(echo $E2E_NUM_FAILED_TEST_SUITES)" >> $GITHUB_OUTPUT
           if [[ ${E2E_NUM_FAILED_TEST_SUITES} -gt 0 ]]; then
             echo "::notice::${E2E_NUM_FAILED_TEST_SUITES} test suite(s) failed in the first run but we will try (it) them again in the second run."
@@ -30,18 +30,19 @@ runs:
       shell: bash
       # Filter failed E2E files from the result JSON file, and re-run them.
       run: |
-        cat "$E2E_RESULT_FILEPATH" | jq '.testResults[] | select(.status == "failed") | .name' | xargs npm run test:e2e -- --NODE_ENV="$NODE_ENV"
+        npm run test:e2e-ci $(cat $E2E_RESULT_FILEPATH | jq -r '[.suites[] | (if has("suites") then .suites[] | .specs[] else .specs[] end) | select(.tests[].status == "unexpected") | .file] | unique | .[]')
 
     # Archive screenshots if any
     - name: Archive e2e test screenshots & logs
-      if: ${{ failure() }}
+      if: ${{ always() }}
       uses: actions/upload-artifact@v4
       with:
           name: wp(${{ env.E2E_WP_VERSION }})-wc(${{ env.E2E_WC_VERSION }})-${{ env.E2E_GROUP }}-${{ env.E2E_BRANCH }}
           path: |
             screenshots
             tests/e2e/screenshots
             tests/e2e/docker/wordpress/wp-content/debug.log
+            tests/e2e-pw/test-results
             ${{ env.E2E_RESULT_FILEPATH }}
           if-no-files-found: ignore
           retention-days: 14

.github/workflows/e2e-pull-request.yml

@@ -30,7 +30,7 @@ env:
   E2E_SLACK_CHANNEL:             ${{ secrets.E2E_SLACK_CHANNEL }}
   E2E_SLACK_TOKEN:               ${{ secrets.E2E_SLACK_TOKEN }}
   E2E_USE_LOCAL_SERVER:          false
-  E2E_RESULT_FILEPATH:           'tests/e2e/results.json'
+  E2E_RESULT_FILEPATH:           'tests/e2e-pw/results.json'
   WCPAY_USE_BUILD_ARTIFACT:      ${{ inputs.wcpay-use-build-artifact }}
   WCPAY_ARTIFACT_DIRECTORY:      'zipfile'
   NODE_ENV:                      'test'
@@ -74,5 +74,9 @@ jobs:
       - name: Setup E2E environment
         uses: ./.github/actions/e2e/env-setup
 
+      - name: Install Playwright
+        shell: bash
+        run: npx playwright install chromium
+
       - name: Run tests, upload screenshots & logs
         uses: ./.github/actions/e2e/run-log-tests

.github/workflows/e2e-test.yml

@@ -22,7 +22,7 @@ env:
   E2E_SLACK_CHANNEL:             ${{ secrets.E2E_SLACK_CHANNEL }}
   E2E_SLACK_TOKEN:               ${{ secrets.E2E_SLACK_TOKEN }}
   E2E_USE_LOCAL_SERVER:          false
-  E2E_RESULT_FILEPATH:           'tests/e2e/results.json'
+  E2E_RESULT_FILEPATH:           'tests/e2e-pw/results.json'
   WC_MIN_SUPPORTED_VERSION:      '7.6.0'
   NODE_ENV:                      'test'
   FORCE_E2E_DEPS_SETUP:          true
@@ -64,6 +64,10 @@ jobs:
       - name: Setup E2E environment
         uses: ./.github/actions/e2e/env-setup
 
+      - name: Install Playwright
+        shell: bash
+        run: npx playwright install chromium
+
       - name: Run tests, upload screenshots & logs
         uses: ./.github/actions/e2e/run-log-tests
 
@@ -116,5 +120,9 @@ jobs:
       - name: Setup E2E environment
         uses: ./.github/actions/e2e/env-setup
 
+      - name: Install Playwright
+        shell: bash
+        run: npx playwright install chromium
+
       - name: Run tests, upload screenshots & logs
         uses: ./.github/actions/e2e/run-log-tests

changelog/dev-10238-update-e2e-tests-all-workflow

@@ -0,0 +1,4 @@
+Significance: patch
+Type: dev
+
+Update Github actions and workflows. Change e2e tests directory structure.

tests/e2e-pw/playwright.config.ts

@@ -9,7 +9,30 @@ import path from 'path';
 config( { path: path.resolve( __dirname, '../e2e/config', '.env' ) } );
 config( { path: path.resolve( __dirname, '../e2e/config', 'local.env' ) } );
 
-const { BASE_URL } = process.env;
+const { BASE_URL, E2E_GROUP, E2E_BRANCH } = process.env;
+
+const validGroups = [ 'wcpay', 'subscriptions' ];
+const validBranches = [ 'merchant', 'shopper' ];
+
+const buildTestDir = ( group: string, branch: string ) => {
+	const baseDir = `\/specs`;
+
+	if ( ! group || ! validGroups.includes( group ) ) {
+		return baseDir;
+	}
+
+	if ( ! branch || ! validBranches.includes( branch ) ) {
+		return `${ baseDir }\/${ group }`;
+	}
+
+	return `${ baseDir }\/${ group }\/${ branch }`;
+};
+
+const getTestMatch = ( group: string, branch: string ) => {
+	const testDir = buildTestDir( group, branch );
+
+	return new RegExp( `${ testDir }\/.*\.spec\.ts` );
+};
 
 /**
  * See https://playwright.dev/docs/test-configuration.
@@ -29,6 +52,7 @@ export default defineConfig( {
 		? [
 				// If running on CI, also use the GitHub Actions reporter
 				[ 'github' ],
+				[ 'json', { outputFile: 'results.json' } ],
 				[ 'html' ],
 		  ]
 		: [ [ 'html', { open: 'never' } ] ],
@@ -43,12 +67,17 @@ export default defineConfig( {
 	},
 	timeout: 120 * 1000, // Default is 30s, somteimes it is not enough for local tests due to long setup.
 	expect: {
-		toHaveScreenshot: { maxDiffPixelRatio: 0.025 },
+		toHaveScreenshot: {
+			maxDiffPixelRatio:
+				process.env.E2E_WC_VERSION === '7.7.0' ? 0.035 : 0.025,
+		},
 		//=* Increase expect timeout to 10 seconds. See https://playwright.dev/docs/test-timeouts#set-expect-timeout-in-the-config.*/
 		timeout: 20 * 1000,
 	},
 	snapshotPathTemplate: '{testDir}/__snapshots__/{testFilePath}/{arg}{ext}',
 
+	testMatch: getTestMatch( E2E_GROUP, E2E_BRANCH ),
+
 	/* Configure projects for major browsers */
 	projects: [
 		{
@@ -58,15 +87,8 @@ export default defineConfig( {
 			dependencies: [ 'setup' ],
 		},
 		{
-			name: 'merchant',
-			use: { ...devices[ 'Desktop Chrome' ] },
-			testDir: './specs/merchant',
-			dependencies: [ 'setup' ],
-		},
-		{
-			name: 'shopper',
+			name: 'chromium',
 			use: { ...devices[ 'Desktop Chrome' ] },
-			testDir: './specs/shopper',
 			dependencies: [ 'setup' ],
 		},
 		// Setup project

tests/e2e-pw/specs/merchant/merchant-subscriptions-renew-action-scheduler.spec.ts → tests/e2e-pw/specs/subscriptions/merchant/merchant-subscriptions-renew-action-scheduler.spec.ts

@@ -6,18 +6,18 @@ import { test, expect } from '@playwright/test';
 /**
  * Internal dependencies
  */
-import { describeif, getMerchant, getShopper } from '../../utils/helpers';
-import * as shopper from '../../utils/shopper';
-import { config } from '../../config/default';
+import { describeif, getMerchant, getShopper } from '../../../utils/helpers';
+import * as shopper from '../../../utils/shopper';
+import { config } from '../../../config/default';
 import {
 	products,
 	shouldRunActionSchedulerTests,
 	shouldRunSubscriptionsTests,
-} from '../../utils/constants';
+} from '../../../utils/constants';
 import {
 	goToActionScheduler,
 	goToSubscriptions,
-} from '../../utils/merchant-navigation';
+} from '../../../utils/merchant-navigation';
 
 // Run the tests if the two 'skip' environment variables are not set.
 describeif( shouldRunSubscriptionsTests && shouldRunActionSchedulerTests )(
@@ -29,8 +29,6 @@ describeif( shouldRunSubscriptionsTests && shouldRunActionSchedulerTests )(
 		const customerBillingConfig =
 			config.addresses[ 'subscriptions-customer' ].billing;
 
-		let subscriptionId: string;
-
 		test.beforeAll( async ( { browser }, { project } ) => {
 			const { shopperPage } = await getShopper(
 				browser,
@@ -48,10 +46,6 @@ describeif( shouldRunSubscriptionsTests && shouldRunActionSchedulerTests )(
 			await expect(
 				shopperPage.getByRole( 'heading', { name: 'Order received' } )
 			).toBeVisible();
-
-			subscriptionId = await shopperPage
-				.getByLabel( 'View subscription number' )
-				.innerText();
 		} );
 
 		test( 'should renew a subscription with action scheduler', async ( {
@@ -81,12 +75,8 @@ describeif( shouldRunSubscriptionsTests && shouldRunActionSchedulerTests )(
 			// Go to Subscriptions and verify the subscription renewal
 			await goToSubscriptions( merchantPage );
 
-			const numericSubscriptionId = subscriptionId.substring( 1 );
-
 			await expect(
-				merchantPage
-					.locator( `#order-${ numericSubscriptionId }` )
-					.getByRole( 'cell', { name: '2', exact: true } )
+				merchantPage.getByRole( 'cell', { name: '2', exact: true } )
 			).toBeVisible();
 		} );
 	}

tests/e2e-pw/specs/merchant/merchant-subscriptions-renew.spec.ts → tests/e2e-pw/specs/subscriptions/merchant/merchant-subscriptions-renew.spec.ts

@@ -6,19 +6,19 @@ import { test, expect } from '@playwright/test';
 /**
  * Internal dependencies
  */
-import RestAPI from '../../utils/rest-api';
-import { config } from '../../config/default';
-import { describeif, getMerchant, getShopper } from '../../utils/helpers';
+import RestAPI from '../../../utils/rest-api';
+import { config } from '../../../config/default';
+import { describeif, getMerchant, getShopper } from '../../../utils/helpers';
 import {
 	emptyCart,
 	fillCardDetails,
 	focusPlaceOrderButton,
 	placeOrder,
 	setupProductCheckout,
-} from '../../utils/shopper';
-import { goToShop } from '../../utils/shopper-navigation';
-import { goToSubscriptionPage } from '../../utils/merchant-navigation';
-import { shouldRunSubscriptionsTests } from '../../utils/constants';
+} from '../../../utils/shopper';
+import { goToShop } from '../../../utils/shopper-navigation';
+import { goToSubscriptionPage } from '../../../utils/merchant-navigation';
+import { shouldRunSubscriptionsTests } from '../../../utils/constants';
 
 const productName = 'Subscription signup fee product';
 const customerBillingConfig =

tests/e2e-pw/specs/merchant/merchant-subscriptions-settings.spec.ts → tests/e2e-pw/specs/subscriptions/merchant/merchant-subscriptions-settings.spec.ts

@@ -2,9 +2,9 @@
  * External dependencies
  */
 import test, { expect } from 'playwright/test';
-import { describeif, useMerchant } from '../../utils/helpers';
-import { shouldRunSubscriptionsTests } from '../../utils/constants';
-import { goToWooCommerceSettings } from '../../utils/merchant-navigation';
+import { describeif, useMerchant } from '../../../utils/helpers';
+import { shouldRunSubscriptionsTests } from '../../../utils/constants';
+import { goToWooCommerceSettings } from '../../../utils/merchant-navigation';
 
 describeif( shouldRunSubscriptionsTests )(
 	'WooCommerce > Settings > Subscriptions',
@@ -21,9 +21,11 @@ describeif( shouldRunSubscriptionsTests )(
 			await expect( menuItem ).toBeVisible();
 
 			// An alternative way to verify the subscriptions menu page is active, avoiding the active tab classname.
-			const heading = await page.getByRole( 'heading', {
-				name: 'Subscriptions',
-			} );
+			const heading = await page
+				.getByRole( 'heading', {
+					name: 'Subscriptions',
+				} )
+				.first();
 			await expect( heading ).toBeVisible();
 		} );
 	}

tests/e2e-pw/specs/shopper/shopper-subscriptions-manage-payments.spec.ts → tests/e2e-pw/specs/subscriptions/shopper/shopper-subscriptions-manage-payments.spec.ts

@@ -6,11 +6,14 @@ import test, { expect, Page } from '@playwright/test';
 /**
  * Internal dependencies
  */
-import { config } from '../../config/default';
-import { products, shouldRunSubscriptionsTests } from '../../utils/constants';
-import { describeif, getShopper } from '../../utils/helpers';
-import * as shopper from '../../utils/shopper';
-import * as navigation from '../../utils/shopper-navigation';
+import { config } from '../../../config/default';
+import {
+	products,
+	shouldRunSubscriptionsTests,
+} from '../../../utils/constants';
+import { describeif, getShopper } from '../../../utils/helpers';
+import * as shopper from '../../../utils/shopper';
+import * as navigation from '../../../utils/shopper-navigation';
 
 const navigateToSubscriptionDetails = async (
 	page: Page,

tests/e2e-pw/specs/shopper/shopper-subscriptions-purchase-free-trial.spec.ts → tests/e2e-pw/specs/subscriptions/shopper/shopper-subscriptions-purchase-free-trial.spec.ts

@@ -6,25 +6,28 @@ import test, { Page, expect } from '@playwright/test';
 /**
  * Internal dependencies
  */
-import { shouldRunSubscriptionsTests } from '../../utils/constants';
-import { describeif, getMerchant, getShopper } from '../../utils/helpers';
-import { config } from '../../config/default';
+import { shouldRunSubscriptionsTests } from '../../../utils/constants';
+import { describeif, getMerchant, getShopper } from '../../../utils/helpers';
+import { config } from '../../../config/default';
 import {
 	confirmCardAuthentication,
 	emptyCart,
 	fillCardDetails,
 	setupCheckout,
-} from '../../utils/shopper';
+} from '../../../utils/shopper';
 import {
 	goToCart,
 	goToProductPageBySlug,
-} from '../../utils/shopper-navigation';
-import { goToOrder, goToSubscriptions } from '../../utils/merchant-navigation';
+} from '../../../utils/shopper-navigation';
+import {
+	goToOrder,
+	goToSubscriptions,
+} from '../../../utils/merchant-navigation';
 import {
 	activateMulticurrency,
 	deactivateMulticurrency,
 	isMulticurrencyEnabled,
-} from '../../utils/merchant';
+} from '../../../utils/merchant';
 
 const nowLocal = new Date();
 const nowUTC = new Date(

tests/e2e-pw/specs/shopper/shopper-subscriptions-purchase-multiple-subscriptions.spec.ts → tests/e2e-pw/specs/subscriptions/shopper/shopper-subscriptions-purchase-multiple-subscriptions.spec.ts

@@ -6,25 +6,25 @@ import test, { Page, expect } from '@playwright/test';
 /**
  * Internal dependencies
  */
-import { shouldRunSubscriptionsTests } from '../../utils/constants';
-import { describeif, getMerchant, getShopper } from '../../utils/helpers';
-import { config } from '../../config/default';
+import { shouldRunSubscriptionsTests } from '../../../utils/constants';
+import { describeif, getMerchant, getShopper } from '../../../utils/helpers';
+import { config } from '../../../config/default';
 import {
 	emptyCart,
 	fillCardDetails,
 	placeOrder,
 	setupProductCheckout,
-} from '../../utils/shopper';
+} from '../../../utils/shopper';
 import {
 	goToShop,
 	goToShopWithCurrency,
 	goToSubscriptions,
-} from '../../utils/shopper-navigation';
+} from '../../../utils/shopper-navigation';
 import {
 	activateMulticurrency,
 	deactivateMulticurrency,
 	restoreCurrencies,
-} from '../../utils/merchant';
+} from '../../../utils/merchant';
 
 const products = {
 	'Subscription no signup fee product': 'subscription-no-signup-fee-product',