Update E2E performance tests to run just from the NPM script

package.json

@@ -37,7 +37,7 @@
     "test:e2e": "./tests/e2e/test-e2e.sh",
     "test:e2e-ci": "npx playwright test --config=tests/e2e/playwright.config.ts --grep-invert @todo",
     "test:e2e-ui": "./tests/e2e/test-e2e-ui.sh",
-    "test:e2e-performance": "NODE_CONFIG_DIR='tests/e2e/config' wp-scripts test-e2e --config tests/e2e/config/jest.performance.config.js",
+    "test:e2e-performance": "./tests/e2e/test-e2e-performance.sh",
     "test:e2e-update-snapshots": "npm run test:e2e -- --update-snapshots",
     "test:update-snapshots": "npm run test:js -- --updateSnapshot",
     "test:php": "./bin/run-tests.sh",

tests/e2e/playwright.config.ts

@@ -73,7 +73,7 @@ export default defineConfig( {
 		video: 'on-first-retry',
 		viewport: { width: 1280, height: 720 },
 	},
-	timeout: 120 * 1000, // Default is 30s, somteimes it is not enough for local tests due to long setup.
+	timeout: 120 * 1000, // Default is 30s, sometimes it is not enough for local tests due to long setup.
 	expect: {
 		toHaveScreenshot: {
 			maxDiffPixelRatio:
@@ -85,6 +85,7 @@ export default defineConfig( {
 	snapshotPathTemplate: '{testDir}/__snapshots__/{testFilePath}/{arg}{ext}',
 
 	testMatch: getTestMatch( E2E_GROUP, E2E_BRANCH ),
+	testIgnore: /specs\/performance/,
 
 	/* Configure projects for major browsers */
 	projects: [

tests/e2e/playwright.performance.config.ts

@@ -0,0 +1,82 @@
+/* eslint-disable @typescript-eslint/naming-convention */
+/**
+ * External dependencies
+ */
+import { defineConfig, devices } from '@playwright/test';
+import { config } from 'dotenv';
+import path from 'path';
+
+config( { path: path.resolve( __dirname, '../e2e/config', '.env' ) } );
+config( { path: path.resolve( __dirname, '../e2e/config', 'local.env' ) } );
+
+const { BASE_URL, NODE_ENV } = process.env;
+
+const getBaseUrl = () => {
+	if ( NODE_ENV === 'atomic' ) {
+		return 'https://wcpaytestecomm.wpcomstaging.com/';
+	}
+
+	return BASE_URL ?? 'http://localhost:8084';
+};
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+export default defineConfig( {
+	testDir: './specs/',
+	/* Run tests in files in parallel */
+	fullyParallel: false,
+	/* Fail the build on CI if you accidentally left test.only in the source code. */
+	forbidOnly: !! process.env.CI,
+	/* Retry on CI only */
+	retries: process.env.CI ? 2 : 0,
+	/* Opt out of parallel tests. */
+	workers: 1,
+	/* Reporters to use. See https://playwright.dev/docs/test-reporters */
+	reporter: process.env.CI
+		? [
+				/* If running on CI, include the dot reporter and JSON reporter. */
+				[ 'dot' ],
+				[ 'json', { outputFile: 'results.json' } ],
+				[ 'html' ],
+		  ]
+		: [ [ 'html', { open: 'never' } ] ],
+	outputDir: './test-results',
+	/* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+	use: {
+		baseURL: getBaseUrl(),
+		screenshot: 'only-on-failure',
+		trace: 'retain-on-failure',
+		video: 'on-first-retry',
+		viewport: { width: 1280, height: 720 },
+	},
+	timeout: 120 * 1000, // Default is 30s, sometimes it is not enough for local tests due to long setup.
+	expect: {
+		toHaveScreenshot: {
+			maxDiffPixelRatio:
+				process.env.E2E_WC_VERSION === '7.7.0' ? 0.035 : 0.025,
+		},
+		//=* Increase expect timeout to 10 seconds. See https://playwright.dev/docs/test-timeouts#set-expect-timeout-in-the-config.*/
+		timeout: 20 * 1000,
+	},
+	snapshotPathTemplate: '{testDir}/__snapshots__/{testFilePath}/{arg}{ext}',
+
+	testMatch: /specs\/performance/,
+
+	/* Configure projects for major browsers */
+	projects: [
+		{
+			name: 'basic',
+			use: { ...devices[ 'Desktop Chrome' ] },
+			testMatch: /basic.spec.ts/,
+			dependencies: [ 'setup' ],
+		},
+		{
+			name: 'chromium',
+			use: { ...devices[ 'Desktop Chrome' ] },
+			dependencies: [ 'setup' ],
+		},
+		// Setup project
+		{ name: 'setup', testMatch: /.*\.setup\.ts/ },
+	],
+} );

tests/e2e/test-e2e-performance.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+echo "🎭 Running Playwright e2e tests in default headless mode, skipping @todo.";
+
+docker compose -f ./tests/e2e/docker-compose.yml run --rm -it --service-ports playwright \
+	npx playwright test --config=tests/e2e/playwright.performance.config.ts --grep-invert @todo "$@"