E2E: drop the prime-logins project and AUTHENTICATE_ACCOUNTS

The setup project logged in as a per-build-type list of accounts before the
suite, so the specs would find cookies rather than all log in at once. The
list is hand written per test group: it drifts from what the specs use, an
account missing from it got no priming at all, and a build type running the
wrong list paid for logins no spec needs. The login lock covers the same
ground without a list.

Removes the project, the env var behind it, the resolver and the reporting
script that read it, the 25 TeamCity parameters setting it and the semicolon
form EXTRA_ENV_VARS grew to carry one. Also the priming half of the legacy
Jest global setup, which has had no specs to prime since the last one moved
to Playwright Test.

Author: lucatume

.teamcity/_self/CalypsoE2ETestsBuildTemplate.kt

@@ -23,11 +23,6 @@ object CalypsoE2ETestsBuildTemplate : Template({
 		param("env.NODE_CONFIG_ENV", "test")
 		param("env.PLAYWRIGHT_BROWSERS_PATH", "0")
 		param("env.LOCALE", "en")
-		// No AUTHENTICATE_ACCOUNTS here on purpose: it names the accounts the prime-logins
-		// setup project logs in as beyond the one the environment resolves to, and that is per
-		// test group, not per template. A build type running a group should set it; leaving it
-		// unset primes every account any group uses, which is safe but slower. See
-		// test/e2e/setup/prime-logins.setup.ts.
 		// required in the CTRF report
 		param("env.BRANCH_NAME", "%teamcity.build.branch%")
 		param("PROJECT", "desktop")
@@ -165,14 +160,9 @@ object CalypsoE2ETestsBuildTemplate : Template({
 			name = "Set extra environment variables"
 			id = "set_extra_env_vars"
 			scriptContent = """
-				# Parse EXTRA_ENV_VARS param (KEY=value pairs) and set as TeamCity env params.
-				# Pairs are separated by semicolons so that a value can hold a comma, as
-				# AUTHENTICATE_ACCOUNTS does; a value with no semicolon keeps the older
-				# comma-separated form, so a saved custom run still parses.
+				# Parse EXTRA_ENV_VARS param (comma-separated KEY=value pairs) and set as TeamCity env params
 				if [[ -n "%EXTRA_ENV_VARS%" ]]; then
-					SEPARATOR=','
-					[[ "%EXTRA_ENV_VARS%" == *";"* ]] && SEPARATOR=';'
-					IFS="${'$'}SEPARATOR" read -ra ENV_PAIRS <<< "%EXTRA_ENV_VARS%"
+					IFS=',' read -ra ENV_PAIRS <<< "%EXTRA_ENV_VARS%"
 					for pair in "${'$'}{ENV_PAIRS[@]}"; do
 						KEY="${'$'}{pair%%=*}"
 						VALUE="${'$'}{pair#*=}"
@@ -200,16 +190,6 @@ object CalypsoE2ETestsBuildTemplate : Template({
 				fi
 				echo "Playwright grep flag: ${'$'}{GREP_FLAG:-(none, running all tests)}"
 
-				# AUTHENTICATE_ACCOUNTS names the accounts this build's group logs in as, so it
-				# is wrong for a build type whose group was adapted away to run everything: hand
-				# priming back its own default list instead. Only that adaptation clears the
-				# flag; a build type selecting its specs through PROJECT leaves TEST_GROUP empty
-				# on purpose and keeps its list.
-				if [[ "%IGNORE_TEST_GROUP_FOR_E2E_CHANGES%" == "true" && -z "${'$'}GREP_FLAG" ]]; then
-					echo "No test group: priming the default accounts instead of AUTHENTICATE_ACCOUNTS"
-					unset AUTHENTICATE_ACCOUNTS
-				fi
-
 				cd test/e2e
 				# Clear any stale teardown-leak markers from a reused checkout before this run.
 				# Recursive over output/: markers should land in output/teardown-leaks, but a

.teamcity/_self/projects/MarTech.kt

@@ -38,7 +38,6 @@ object ToSAcceptanceTracking : BuildType({
 		param("TEST_GROUP", "@legal")
 		param("CALYPSO_BASE_URL", "https://wordpress.com")
 		param("DASHBOARD_BASE_URL", "https://my.wordpress.com")
-		param("env.AUTHENTICATE_ACCOUNTS", "")
 	}
 
 	steps {

.teamcity/_self/projects/WPComTests.kt

@@ -23,13 +23,6 @@ import jetbrains.buildServer.configs.kotlin.v2019_2.buildSteps.exec
 import jetbrains.buildServer.configs.kotlin.v2019_2.buildSteps.ScriptBuildStep
 import jetbrains.buildServer.configs.kotlin.v2019_2.matrix
 
-// Accounts the @jetpack-wpcom-integration specs log in as on top of the one the run's own
-// environment resolves to, which the prime-logins setup project adds by itself. On Simple
-// that is the Blaze spec's free plan site plus the site editor's own staging site; on Atomic
-// both of those resolve to the account the environment already gives us.
-const val jetpackWpcomIntegrationSimpleAccounts = "simpleSiteFreePlanUser,jetpackStagingFseUser"
-const val jetpackWpcomIntegrationAtomicAccounts = ""
-
 object WPComTests : Project({
 	id("WPComTests")
 	name = "WPCom Tests"
@@ -116,17 +109,6 @@ fun gutenbergPlaywrightBuildType( targetDevice: String, buildUuid: String, atomi
 				checked = "true",
 				unchecked = "false"
 			)
-			// The account this build runs against is added by the prime-logins project itself.
-			// On top of it the @gutenberg specs log in as defaultUser to read a published post
-			// as a second user, as the site editor account for this leg (the environment key
-			// carries no variant, so that one is never derived), and, on stable Gutenberg on a
-			// Simple site only, resolve their criteria overrides to simpleSitePersonalPlanUser.
-			param("env.AUTHENTICATE_ACCOUNTS", when {
-				atomic && (edge || nightly) -> "defaultUser,siteEditorAtomicSiteEdgeUser"
-				atomic -> "defaultUser,siteEditorAtomicSiteUser"
-				edge || nightly -> "defaultUser,siteEditorSimpleSiteEdgeUser"
-				else -> "defaultUser,simpleSitePersonalPlanUser,siteEditorSimpleSiteUser"
-			})
 			if (atomic) {
 				param("env.TEST_ON_ATOMIC", "true")
 				// Overrides the inherited max workers settings and sets it to not run any tests in parallel.
@@ -217,7 +199,6 @@ fun jetpackSimpleDeploymentE2eBuildType( targetDevice: String, buildUuid: String
 			calypsoBaseUrlParam()
 			param("env.VIEWPORT_NAME", "$targetDevice")
 			param("env.JETPACK_TARGET", "wpcom-deployment")
-			param("env.AUTHENTICATE_ACCOUNTS", jetpackWpcomIntegrationSimpleAccounts)
 		}
 
 		steps {
@@ -281,7 +262,6 @@ fun jetpackAtomicDeploymentE2eBuildType( targetDevice: String, buildUuid: String
 			param("env.VIEWPORT_NAME", "$targetDevice")
 			param("env.JETPACK_TARGET", "wpcom-deployment")
 			param("env.TEST_ON_ATOMIC", "true")
-			param("env.AUTHENTICATE_ACCOUNTS", jetpackWpcomIntegrationAtomicAccounts)
 		}
 
 		steps {
@@ -357,7 +337,6 @@ fun jetpackAtomicBuildSmokeE2eBuildType( targetDevice: String, buildUuid: String
 			param("env.JETPACK_TARGET", "wpcom-deployment")
 			param("env.TEST_ON_ATOMIC", "true")
 			param("env.ATOMIC_VARIATION", "mixed")
-			param("env.AUTHENTICATE_ACCOUNTS", jetpackWpcomIntegrationAtomicAccounts)
 		}
 
 		steps {
@@ -411,7 +390,6 @@ private object I18NTests : BuildType({
 		param("CALYPSO_BASE_URL", "https://wordpress.com")
 		param("DASHBOARD_BASE_URL", "https://my.wordpress.com")
 		param("env.E2E_CTRF_APP_NAME", "i18n (calypso)")
-		param("env.AUTHENTICATE_ACCOUNTS", "i18nUser")
 	}
 
 	features {
@@ -456,7 +434,6 @@ private object P2E2ETests : BuildType({
 		param("CALYPSO_BASE_URL", "https://wpcalypso.wordpress.com")
 		param("DASHBOARD_BASE_URL", "https://my.wordpress.com")
 		param("env.E2E_CTRF_APP_NAME", "p2 (calypso)")
-		param("env.AUTHENTICATE_ACCOUNTS", "p2User")
 	}
 
 	features {
@@ -508,11 +485,6 @@ private object GutenbergPlaywrightTests : BuildType({
 		param("CALYPSO_BASE_URL", "https://wordpress.com")
 		param("DASHBOARD_BASE_URL", "https://my.wordpress.com")
 		param("env.E2E_CTRF_APP_NAME", "gutenberg (calypso)")
-		// The Simple Production leg; the others override this through EXTRA_ENV_VARS below.
-		param(
-			"env.AUTHENTICATE_ACCOUNTS",
-			"defaultUser,simpleSitePersonalPlanUser,siteEditorSimpleSiteUser"
-		)
 		password("GB_E2E_ANNOUNCEMENT_SLACK_API_TOKEN", "credentialsJSON:8196e9b8-cf0a-4ab5-9547-95145134f04a", display = ParameterDisplay.HIDDEN);
 		// Uncomment the following to route it to the test channel, don't forget to change the reference in the exec() calls below, too.
 		// Ask someone from the Team Calypso Platform to know what these channels are. They are also available in the source for `announce.sh` (par of Gutenbot).
@@ -545,15 +517,12 @@ private object GutenbergPlaywrightTests : BuildType({
 				value("desktop", label = "Desktop"),
 				value("mobile", label = "Mobile"),
 			))
-			// Each leg carries the site editor account its own environment resolves to, which
-			// the prime-logins project cannot derive: the environment key holds no variant.
-			// The Simple Production leg takes the build's own AUTHENTICATE_ACCOUNTS above.
 			param("EXTRA_ENV_VARS", listOf(
 				value("", label = "Simple Production"),
-				value("GUTENBERG_EDGE=true;AUTHENTICATE_ACCOUNTS=defaultUser,siteEditorSimpleSiteEdgeUser", label = "Simple Edge"),
-				value("TEST_ON_ATOMIC=true;PW_WORKERS=1;AUTHENTICATE_ACCOUNTS=defaultUser,siteEditorAtomicSiteUser", label = "Atomic Production"),
-				value("TEST_ON_ATOMIC=true;GUTENBERG_EDGE=true;PW_WORKERS=1;AUTHENTICATE_ACCOUNTS=defaultUser,siteEditorAtomicSiteEdgeUser", label = "Atomic Edge"),
-				value("TEST_ON_ATOMIC=true;GUTENBERG_NIGHTLY=true;PW_WORKERS=1;AUTHENTICATE_ACCOUNTS=defaultUser,siteEditorAtomicSiteEdgeUser", label = "Atomic Nightly"),
+				value("GUTENBERG_EDGE=true", label = "Simple Edge"),
+				value("TEST_ON_ATOMIC=true,PW_WORKERS=1", label = "Atomic Production"),
+				value("TEST_ON_ATOMIC=true,GUTENBERG_EDGE=true,PW_WORKERS=1", label = "Atomic Edge"),
+				value("TEST_ON_ATOMIC=true,GUTENBERG_NIGHTLY=true,PW_WORKERS=1", label = "Atomic Nightly"),
 			))
 		}
 		notifyAllFailuresAndFirstSuccess("#gutenberg-e2e")
@@ -580,7 +549,6 @@ private object JetpackE2ETestsBuildTemplate : Template({
 		param("CALYPSO_BASE_URL", "https://wordpress.com")
 		param("env.E2E_CTRF_APP_NAME", "jetpack (calypso)")
 		param("env.JETPACK_TARGET", "wpcom-deployment")
-		param("env.AUTHENTICATE_ACCOUNTS", jetpackWpcomIntegrationSimpleAccounts)
 	}
 
 	features {
@@ -629,7 +597,6 @@ private object JetpackAtomicE2ETests : BuildType({
 		param("PROJECT", "desktop")
 		param("env.TEST_ON_ATOMIC", "true")
 		param("env.PW_WORKERS", "5")
-		param("env.AUTHENTICATE_ACCOUNTS", jetpackWpcomIntegrationAtomicAccounts)
 	}
 
 	features {
@@ -659,6 +626,5 @@ private object JetpackAtomicSmokeE2ETests : BuildType({
 		param("env.TEST_ON_ATOMIC", "true")
 		param("env.PW_WORKERS", "14")
 		param("env.ATOMIC_VARIATION", "mixed")
-		param("env.AUTHENTICATE_ACCOUNTS", jetpackWpcomIntegrationAtomicAccounts)
 	}
 })

.teamcity/_self/projects/WebApp.kt

@@ -984,9 +984,6 @@ object PlaywrightTestPRMatrix : BuildType({
 		param("TEST_GROUP", "@calypso-pr")
         param("DOCKER_IMAGE_BUILD_NUMBER", "${BuildDockerImage.depParamRefs.buildNumber}")
 		param("IGNORE_TEST_GROUP_FOR_E2E_CHANGES", "true")
-		// calypsoPreReleaseUser is left out: the only @calypso-pr spec that logs in as it runs
-		// on trunk alone, and this build type never builds trunk.
-		param("env.AUTHENTICATE_ACCOUNTS", "atomicUser,commentingUser,defaultUser,gutenbergSimpleSiteUser,notificationsUser,siteEditorSimpleSiteUser,simpleSiteFreePlanUser,simpleSitePersonalPlanUser")
 	}
 
 	features {
@@ -1039,7 +1036,6 @@ object PlaywrightTestPreReleaseMatrix : BuildType({
 		text("TEST_GROUP", "@calypso-release")
 		param("CALYPSO_BASE_URL", "https://wpcalypso.wordpress.com")
 		param("DASHBOARD_BASE_URL", "https://my.wordpress.com")
-		param("env.AUTHENTICATE_ACCOUNTS", "atomicUser,calypsoPreReleaseUser,defaultUser,simpleSiteFreePlanUser,simpleSitePersonalPlanUser")
 	}
 
 	features {
@@ -1076,10 +1072,6 @@ object PlaywrightTestDashboardPRMatrix : BuildType({
 	params {
 		param("TEST_GROUP", "@dashboard-pr")
 		param("DOCKER_IMAGE_BUILD_NUMBER", "${BuildDockerImage.depParamRefs.buildNumber}")
-		// Every @dashboard-pr spec takes the account this environment resolves to, and
-		// nothing else. Naming it costs no extra login: prime-logins adds the resolved
-		// account to whatever is listed here and logs in as each account once.
-		param("env.AUTHENTICATE_ACCOUNTS", "gutenbergSimpleSiteUser")
 	}
 
 	features {
@@ -1134,8 +1126,6 @@ object PlaywrightTestA4APRMatrix : BuildType({
 	params {
 		param("TEST_GROUP", "@a8c-for-agencies")
 		param("DOCKER_IMAGE_BUILD_NUMBER", "${BuildDockerImage.depParamRefs.buildNumber}")
-		// The @a8c-for-agencies specs sign up their own users and log in as no test account.
-		param("env.AUTHENTICATE_ACCOUNTS", "")
 	}
 
 	features {

packages/calypso-e2e/src/env-variables.ts

@@ -2,17 +2,14 @@
 import crypto from 'crypto';
 import path from 'path';
 import { getMag16Locales, getViewports } from './data-helper';
-import { TEST_ACCOUNT_NAMES } from './secrets';
 import { SupportedEnvVariables, JetpackTarget, AtomicVariation } from './types/env-variables.types';
-import { TestAccountName } from '.';
 
 class EnvVariables implements SupportedEnvVariables {
 	private _defaultEnvVariables: SupportedEnvVariables = {
 		A8C_FOR_AGENCIES_URL: 'https://agencies.automattic.com',
 		ALLURE_RESULTS_PATH: '',
 		ARTIFACTS_PATH: path.join( process.cwd(), 'results' ),
 		ATOMIC_VARIATION: 'default',
-		AUTHENTICATE_ACCOUNTS: [],
 		BROWSER_NAME: 'chromium',
 		CALYPSO_BASE_URL: `http://calypso.localhost:${ process.env.PORT || 3000 }`,
 		COBLOCKS_EDGE: false,
@@ -111,22 +108,6 @@ class EnvVariables implements SupportedEnvVariables {
 			: this._defaultEnvVariables.COBLOCKS_EDGE;
 	}
 
-	get AUTHENTICATE_ACCOUNTS(): TestAccountName[] {
-		const value = process.env.AUTHENTICATE_ACCOUNTS;
-		if ( ! value ) {
-			return this._defaultEnvVariables.AUTHENTICATE_ACCOUNTS;
-		}
-
-		const parsedAccounts: TestAccountName[] = value.split( ',' ) as TestAccountName[];
-		const supportedValues = new Set< TestAccountName >( TEST_ACCOUNT_NAMES );
-		if ( ! parsedAccounts.every( ( account ) => supportedValues.has( account ) ) ) {
-			throw new Error(
-				`Unknown AUTHENTICATE_ACCOUNTS value: ${ value }.\nSupported values: ${ TEST_ACCOUNT_NAMES }`
-			);
-		}
-		return parsedAccounts;
-	}
-
 	get COOKIES_PATH(): string {
 		const value = process.env.COOKIES_PATH;
 		return value ? value : this._defaultEnvVariables.COOKIES_PATH;

packages/calypso-e2e/src/jest-playwright-config/global-setup.ts

@@ -1,11 +1,7 @@
-/* eslint-disable require-jsdoc */
-import { Browser, chromium } from 'playwright';
 import envVariables from '../env-variables';
-import { TestAccount } from '../lib/test-account';
-import pwConfig from './playwright-config';
 
 export default async (): Promise< void > => {
-	const { AUTHENTICATE_ACCOUNTS, CALYPSO_BASE_URL } = envVariables;
+	const { CALYPSO_BASE_URL } = envVariables;
 
 	// Checks whether the Calypso server is running so we can exit early with a help message.
 	// Don't run in CI because there seems to be a timing issue about whether calypso.live will be ready in time.
@@ -32,62 +28,4 @@ export default async (): Promise< void > => {
 			process.exit( 1 );
 		}
 	}
-
-	// If PWDEBUG mode is enabled (stepping through each step)
-	// don't execute the cookie refresh.
-	if ( process.env.PWDEBUG ) {
-		return;
-	}
-
-	// If the list of accounts for which to pre-authenticate and save cookies
-	// for is empty, then don't run.
-	if ( AUTHENTICATE_ACCOUNTS.length === 0 ) {
-		return;
-	}
-
-	const browser = await chromium.launch( {
-		...pwConfig.launchOptions,
-		headless: true,
-	} );
-
-	await Promise.all(
-		AUTHENTICATE_ACCOUNTS.map( async ( accountName ) => {
-			const testAccount = new TestAccount( accountName );
-			if ( await testAccount.hasFreshAuthCookies() ) {
-				return;
-			}
-
-			await loginAndSaveCookiesWithRetry( testAccount, browser );
-		} )
-	);
-
-	await browser.close();
 };
-
-async function loginAndSaveCookiesWithRetry( testAccount: TestAccount, browser: Browser ) {
-	const MAX_ATTEMPTS = 2;
-	let numberOfAttempts = 0;
-
-	let error: Error | undefined;
-	while ( numberOfAttempts < MAX_ATTEMPTS ) {
-		try {
-			return await loginAndSaveCookies( testAccount, browser );
-		} catch ( err ) {
-			numberOfAttempts++;
-			error = err as Error;
-		}
-	}
-
-	throw error;
-}
-
-async function loginAndSaveCookies( testAccount: TestAccount, browser: Browser ) {
-	const page = await browser.newPage( pwConfig.contextOptions );
-	page.setDefaultTimeout( envVariables.TIMEOUT );
-	try {
-		await testAccount.logInViaLoginPage( page );
-		await testAccount.saveAuthCookies( page.context() );
-	} finally {
-		await page.close();
-	}
-}

packages/calypso-e2e/src/lib/utils/index.ts

@@ -1,7 +1,6 @@
 /* Utilities */
 export * from './validate-translations';
 export * from './get-test-account-by-feature';
-export * from './accounts-to-prime';
 export * from './translate';
 export * from './social-connections-manager';
 

packages/calypso-e2e/src/types/env-variables.types.ts

@@ -1,5 +1,3 @@
-import { TestAccountName } from '../';
-
 export type ViewportName = string;
 export type TestLocales = string[];
 export type JetpackTarget = 'remote-site' | 'wpcom-production' | 'wpcom-deployment';
@@ -18,7 +16,6 @@ export interface SupportedEnvVariables {
 	readonly ALLURE_RESULTS_PATH: string;
 	readonly ARTIFACTS_PATH: string;
 	readonly ATOMIC_VARIATION: AtomicVariation;
-	readonly AUTHENTICATE_ACCOUNTS: TestAccountName[];
 	readonly BROWSER_NAME: string;
 	readonly CALYPSO_BASE_URL: string;
 	readonly COBLOCKS_EDGE: boolean;

test/e2e/AGENTS.md

@@ -141,16 +141,7 @@ test( 'Test', async ( { pageLogin, componentSidebar } ) => {
 
 ### Available Fixtures
 
-**Accounts**: one fixture per key of `fixtureAccounts` in [`lib/pw-base.ts`](lib/pw-base.ts), plus `accountGivenByEnvironment` and `accountSMS`. Declaring an account fixture logs in as it, whether or not the test body uses it, so take only the ones the test needs. To have a build log in as an account before the suite instead of during it, add it to that build type's `AUTHENTICATE_ACCOUNTS` parameter; see [`setup/prime-logins.setup.ts`](setup/prime-logins.setup.ts).
-
-To see what every build type logs in as before its suite, without starting a build:
-
-```bash
-# From the repository root. Regenerate only after a .teamcity change; the DSL needs JDK 17.
-( cd .teamcity && JAVA_HOME=$(/usr/libexec/java_home -v 17) mvn -q teamcity-configs:generate )
-yarn workspace @automattic/calypso-e2e build
-( cd test/e2e && node bin/primed-accounts.js )
-```
+**Accounts**: one fixture per key of `fixtureAccounts` in [`lib/pw-base.ts`](lib/pw-base.ts), plus `accountGivenByEnvironment` and `accountSMS`. Declaring an account fixture logs in as it, whether or not the test body uses it, so take only the ones the test needs. A run logs in as each account once: the first worker to reach it logs in and saves the cookies the rest load.
 
 **Pages/Components**: Follow naming conventions: