diff --git a/.teamcity/_self/CalypsoE2ETestsBuildTemplate.kt b/.teamcity/_self/CalypsoE2ETestsBuildTemplate.kt index 0439ef656de5..e0d55337a5d7 100644 --- a/.teamcity/_self/CalypsoE2ETestsBuildTemplate.kt +++ b/.teamcity/_self/CalypsoE2ETestsBuildTemplate.kt @@ -23,10 +23,6 @@ object CalypsoE2ETestsBuildTemplate : Template({ param("env.NODE_CONFIG_ENV", "test") param("env.PLAYWRIGHT_BROWSERS_PATH", "0") param("env.LOCALE", "en") - // No AUTHENTICATE_ACCOUNTS on purpose: setting it replaces the whole list the - // prime-logins setup project logs in as, so a value here would skip every account it - // doesn't name. Set it only on a build type running a narrow group, or to an empty - // value to skip priming. See test/e2e/setup/prime-logins.setup.ts. // required in the CTRF report param("env.BRANCH_NAME", "%teamcity.build.branch%") param("PROJECT", "desktop") diff --git a/.teamcity/_self/projects/MarTech.kt b/.teamcity/_self/projects/MarTech.kt index 6c4cfff6a7c2..cc8dd8a823b8 100644 --- a/.teamcity/_self/projects/MarTech.kt +++ b/.teamcity/_self/projects/MarTech.kt @@ -38,7 +38,6 @@ object ToSAcceptanceTracking : BuildType({ param("TEST_GROUP", "@legal") param("CALYPSO_BASE_URL", "https://wordpress.com") param("DASHBOARD_BASE_URL", "https://my.wordpress.com") - param("env.AUTHENTICATE_ACCOUNTS", "") } steps { diff --git a/.teamcity/_self/projects/WPComTests.kt b/.teamcity/_self/projects/WPComTests.kt index 80da7f5c508c..080f795f1c82 100644 --- a/.teamcity/_self/projects/WPComTests.kt +++ b/.teamcity/_self/projects/WPComTests.kt @@ -109,7 +109,6 @@ fun gutenbergPlaywrightBuildType( targetDevice: String, buildUuid: String, atomi checked = "true", unchecked = "false" ) - param("env.AUTHENTICATE_ACCOUNTS", "gutenbergSimpleSiteEdgeUser,gutenbergSimpleSiteUser,coBlocksSimpleSiteEdgeUser,simpleSitePersonalPlanUser,gutenbergAtomicSiteUser,gutenbergAtomicSiteEdgeUser,gutenbergAtomicSiteEdgeNightliesUser") if (atomic) { param("env.TEST_ON_ATOMIC", "true") // Overrides the inherited max workers settings and sets it to not run any tests in parallel. @@ -391,7 +390,6 @@ private object I18NTests : BuildType({ param("CALYPSO_BASE_URL", "https://wordpress.com") param("DASHBOARD_BASE_URL", "https://my.wordpress.com") param("env.E2E_CTRF_APP_NAME", "i18n (calypso)") - param("env.AUTHENTICATE_ACCOUNTS", "i18nUser") } features { @@ -436,7 +434,6 @@ private object P2E2ETests : BuildType({ param("CALYPSO_BASE_URL", "https://wpcalypso.wordpress.com") param("DASHBOARD_BASE_URL", "https://my.wordpress.com") param("env.E2E_CTRF_APP_NAME", "p2 (calypso)") - param("env.AUTHENTICATE_ACCOUNTS", "p2User") } features { @@ -488,7 +485,6 @@ private object GutenbergPlaywrightTests : BuildType({ param("CALYPSO_BASE_URL", "https://wordpress.com") param("DASHBOARD_BASE_URL", "https://my.wordpress.com") param("env.E2E_CTRF_APP_NAME", "gutenberg (calypso)") - param("env.AUTHENTICATE_ACCOUNTS", "gutenbergSimpleSiteEdgeUser,gutenbergSimpleSiteUser,simpleSitePersonalPlanUser,gutenbergAtomicSiteUser,gutenbergAtomicSiteEdgeUser,gutenbergAtomicSiteEdgeNightliesUser") password("GB_E2E_ANNOUNCEMENT_SLACK_API_TOKEN", "credentialsJSON:8196e9b8-cf0a-4ab5-9547-95145134f04a", display = ParameterDisplay.HIDDEN); // Uncomment the following to route it to the test channel, don't forget to change the reference in the exec() calls below, too. // Ask someone from the Team Calypso Platform to know what these channels are. They are also available in the source for `announce.sh` (par of Gutenbot). diff --git a/packages/calypso-e2e/src/env-variables.ts b/packages/calypso-e2e/src/env-variables.ts index 5c5831644067..27c03384ad57 100644 --- a/packages/calypso-e2e/src/env-variables.ts +++ b/packages/calypso-e2e/src/env-variables.ts @@ -2,9 +2,7 @@ import crypto from 'crypto'; import path from 'path'; import { getMag16Locales, getViewports } from './data-helper'; -import { TEST_ACCOUNT_NAMES } from './secrets'; import { SupportedEnvVariables, JetpackTarget, AtomicVariation } from './types/env-variables.types'; -import { TestAccountName } from '.'; class EnvVariables implements SupportedEnvVariables { private _defaultEnvVariables: SupportedEnvVariables = { @@ -12,7 +10,6 @@ class EnvVariables implements SupportedEnvVariables { ALLURE_RESULTS_PATH: '', ARTIFACTS_PATH: path.join( process.cwd(), 'results' ), ATOMIC_VARIATION: 'default', - AUTHENTICATE_ACCOUNTS: [], BROWSER_NAME: 'chromium', CALYPSO_BASE_URL: `http://calypso.localhost:${ process.env.PORT || 3000 }`, COBLOCKS_EDGE: false, @@ -111,22 +108,6 @@ class EnvVariables implements SupportedEnvVariables { : this._defaultEnvVariables.COBLOCKS_EDGE; } - get AUTHENTICATE_ACCOUNTS(): TestAccountName[] { - const value = process.env.AUTHENTICATE_ACCOUNTS; - if ( ! value ) { - return this._defaultEnvVariables.AUTHENTICATE_ACCOUNTS; - } - - const parsedAccounts: TestAccountName[] = value.split( ',' ) as TestAccountName[]; - const supportedValues = new Set< TestAccountName >( TEST_ACCOUNT_NAMES ); - if ( ! parsedAccounts.every( ( account ) => supportedValues.has( account ) ) ) { - throw new Error( - `Unknown AUTHENTICATE_ACCOUNTS value: ${ value }.\nSupported values: ${ TEST_ACCOUNT_NAMES }` - ); - } - return parsedAccounts; - } - get COOKIES_PATH(): string { const value = process.env.COOKIES_PATH; return value ? value : this._defaultEnvVariables.COOKIES_PATH; diff --git a/packages/calypso-e2e/src/jest-playwright-config/global-setup.ts b/packages/calypso-e2e/src/jest-playwright-config/global-setup.ts index 74f2d5322a04..827bda54ef25 100644 --- a/packages/calypso-e2e/src/jest-playwright-config/global-setup.ts +++ b/packages/calypso-e2e/src/jest-playwright-config/global-setup.ts @@ -1,11 +1,7 @@ -/* eslint-disable require-jsdoc */ -import { Browser, chromium } from 'playwright'; import envVariables from '../env-variables'; -import { TestAccount } from '../lib/test-account'; -import pwConfig from './playwright-config'; export default async (): Promise< void > => { - const { AUTHENTICATE_ACCOUNTS, CALYPSO_BASE_URL } = envVariables; + const { CALYPSO_BASE_URL } = envVariables; // Checks whether the Calypso server is running so we can exit early with a help message. // Don't run in CI because there seems to be a timing issue about whether calypso.live will be ready in time. @@ -32,62 +28,4 @@ export default async (): Promise< void > => { process.exit( 1 ); } } - - // If PWDEBUG mode is enabled (stepping through each step) - // don't execute the cookie refresh. - if ( process.env.PWDEBUG ) { - return; - } - - // If the list of accounts for which to pre-authenticate and save cookies - // for is empty, then don't run. - if ( AUTHENTICATE_ACCOUNTS.length === 0 ) { - return; - } - - const browser = await chromium.launch( { - ...pwConfig.launchOptions, - headless: true, - } ); - - await Promise.all( - AUTHENTICATE_ACCOUNTS.map( async ( accountName ) => { - const testAccount = new TestAccount( accountName ); - if ( await testAccount.hasFreshAuthCookies() ) { - return; - } - - await loginAndSaveCookiesWithRetry( testAccount, browser ); - } ) - ); - - await browser.close(); }; - -async function loginAndSaveCookiesWithRetry( testAccount: TestAccount, browser: Browser ) { - const MAX_ATTEMPTS = 2; - let numberOfAttempts = 0; - - let error: Error | undefined; - while ( numberOfAttempts < MAX_ATTEMPTS ) { - try { - return await loginAndSaveCookies( testAccount, browser ); - } catch ( err ) { - numberOfAttempts++; - error = err as Error; - } - } - - throw error; -} - -async function loginAndSaveCookies( testAccount: TestAccount, browser: Browser ) { - const page = await browser.newPage( pwConfig.contextOptions ); - page.setDefaultTimeout( envVariables.TIMEOUT ); - try { - await testAccount.logInViaLoginPage( page ); - await testAccount.saveAuthCookies( page.context() ); - } finally { - await page.close(); - } -} diff --git a/packages/calypso-e2e/src/lib/test-account.ts b/packages/calypso-e2e/src/lib/test-account.ts index fe082bd06a21..24c454fb40d8 100644 --- a/packages/calypso-e2e/src/lib/test-account.ts +++ b/packages/calypso-e2e/src/lib/test-account.ts @@ -1,3 +1,4 @@ +import { randomUUID } from 'crypto'; import fs from 'fs/promises'; import path from 'path'; import chalk from 'chalk'; @@ -13,6 +14,26 @@ import { SidebarComponent } from './components/sidebar-component'; import { LoginPage } from './pages/login-page'; import type { TestAccountCredentials } from '../secrets'; +const LOCK_POLL_MS = 500; +const LOCK_HEARTBEAT_MS = 10 * 1000; +const LOCK_STALE_MS = 45 * 1000; +const LOCK_WAIT_MS = 90 * 1000; +const LOCK_GUARD_HEARTBEAT_MS = 2 * 1000; +const LOCK_GUARD_STALE_MS = 10 * 1000; +const LOCK_GUARD_WAIT_MS = 15 * 1000; +const LOGIN_ATTEMPTS = 2; + +type LoginLock = { + assertOwned: () => Promise< void >; + runIfOwned: < T >( action: () => Promise< T > ) => Promise< T >; + release: () => Promise< void >; +}; + +/** Checks whether a Node error has one of the expected codes. */ +function hasErrorCode( error: unknown, ...codes: string[] ): boolean { + return codes.includes( ( error as NodeJS.ErrnoException ).code ?? '' ); +} + /** * Represents the WPCOM test account. */ @@ -53,15 +74,11 @@ export class TestAccount { { url, waitUntilStable = true }: { url?: string | RegExp; waitUntilStable?: boolean } = {} ): Promise< void > { const browserContext = page.context(); - await browserContext.clearCookies(); - if ( await this.hasFreshAuthCookies() ) { - this.log( 'Found fresh cookies, skipping log in' ); + if ( ! ( await this.ensureFreshAuthCookies( page ) ) ) { + await browserContext.clearCookies(); await browserContext.addCookies( await this.getAuthCookies() ); await page.goto( getCalypsoURL( '/' ) ); - } else { - this.log( 'Logging in via Login Page' ); - await this.logInViaLoginPage( page ); } if ( url ) { @@ -73,6 +90,369 @@ export class TestAccount { } } + /** + * Makes sure a fresh cookies file exists for this account, logging in on the given + * page if it doesn't. + * + * Every worker in a run shares one cookies file per account through COOKIES_PATH, and + * a CI build starts with none of them. Left alone, the workers all miss at once and log + * in concurrently, which is where most of the CI login timeouts come from. Instead the + * first worker to create the account's lock directory logs in and writes the cookies, and + * the rest wait for the cookies: each account is logged in once per run, not once per + * worker. + * + * @param {Page} page Page to log in on, if this worker is the one that logs in. + * @returns {Promise} Whether the login happened on this page, as opposed to + * the cookies having been left by another worker. + */ + async ensureFreshAuthCookies( page: Page ): Promise< boolean > { + const deadline = Date.now() + LOCK_WAIT_MS; + + for (;;) { + if ( await this.hasFreshAuthCookies() ) { + this.log( 'Found fresh cookies, skipping log in' ); + return false; + } + + const loginLock = await this.acquireLoginLock(); + if ( ! loginLock ) { + if ( Date.now() >= deadline ) { + if ( await this.hasFreshAuthCookies() ) { + return false; + } + throw new Error( `Timed out waiting for login lock for ${ this.accountName }` ); + } + await new Promise( ( resolve ) => setTimeout( resolve, LOCK_POLL_MS ) ); + continue; + } + + let loggedIn = false; + let operationError: unknown; + try { + if ( await this.hasFreshAuthCookies() ) { + this.log( 'Found fresh cookies after acquiring login lock' ); + } else { + for ( let attempt = 1; attempt <= LOGIN_ATTEMPTS; attempt++ ) { + await page.context().clearCookies(); + this.log( `Logging in via Login Page (attempt ${ attempt })` ); + try { + await this.logInViaLoginPage( page ); + break; + } catch ( error ) { + await loginLock.assertOwned(); + if ( attempt === LOGIN_ATTEMPTS ) { + throw error; + } + } + } + await loginLock.runIfOwned( () => this.saveAuthCookies( page.context() ) ); + loggedIn = true; + } + } catch ( error ) { + operationError = error; + } + + let releaseError: unknown; + try { + await loginLock.release(); + } catch ( error ) { + releaseError = error; + } + + if ( operationError && releaseError ) { + throw new AggregateError( + [ operationError, releaseError ], + `Login and lock release both failed for ${ this.accountName }` + ); + } + if ( operationError ) { + throw operationError; + } + if ( releaseError ) { + throw releaseError; + } + return loggedIn; + } + } + + /** Tries to acquire this account's login lock. */ + private async acquireLoginLock(): Promise< LoginLock | null > { + const lockPath = this.getLoginLockPath(); + await fs.mkdir( path.dirname( lockPath ), { recursive: true } ); + + const releaseGuard = await this.acquireLoginLockGuard(); + if ( ! releaseGuard ) { + return null; + } + + try { + try { + await fs.mkdir( lockPath ); + return await this.createOwnedLoginLock(); + } catch ( error ) { + if ( ! hasErrorCode( error, 'EEXIST' ) ) { + throw error; + } + } + + let mtimeMs: number; + try { + ( { mtimeMs } = await fs.stat( lockPath ) ); + } catch ( error ) { + if ( hasErrorCode( error, 'ENOENT' ) ) { + return null; + } + throw error; + } + + if ( Date.now() < mtimeMs + LOCK_STALE_MS ) { + return null; + } + + const stalePath = `${ lockPath }.${ randomUUID() }.stale`; + try { + await fs.rename( lockPath, stalePath ); + } catch ( error ) { + if ( hasErrorCode( error, 'ENOENT' ) ) { + return null; + } + throw error; + } + await fs.rm( stalePath, { force: true, recursive: true } ); + try { + await fs.mkdir( lockPath ); + } catch ( error ) { + if ( hasErrorCode( error, 'EEXIST' ) ) { + return null; + } + throw error; + } + this.log( 'Took over a stale login lock' ); + return await this.createOwnedLoginLock(); + } finally { + await releaseGuard(); + } + } + + /** Creates an ownership marker and heartbeat for an acquired lock directory. */ + private async createOwnedLoginLock(): Promise< LoginLock > { + const lockPath = this.getLoginLockPath(); + const ownerPath = path.join( lockPath, `${ process.pid }-${ randomUUID() }.owner` ); + try { + await fs.writeFile( ownerPath, '' ); + } catch ( error ) { + try { + await fs.rm( ownerPath, { force: true } ); + await fs.rmdir( lockPath ); + } catch ( cleanupError ) { + throw new AggregateError( + [ error, cleanupError ], + `Creating and cleaning up the login lock both failed for ${ this.accountName }` + ); + } + throw error; + } + + let heartbeatError: unknown; + let heartbeatPromise = Promise.resolve(); + + const assertOwnerMarker = async () => { + try { + await fs.stat( ownerPath ); + } catch ( error ) { + if ( hasErrorCode( error, 'ENOENT' ) ) { + throw new Error( `Lost login lock for ${ this.accountName }`, { cause: error } ); + } + throw error; + } + }; + + const runIfOwned = async < T >( action: () => Promise< T > ): Promise< T > => { + await heartbeatPromise; + const releaseGuard = await this.waitForLoginLockGuard(); + try { + if ( heartbeatError ) { + throw new Error( `Lost login lock for ${ this.accountName }`, { + cause: heartbeatError, + } ); + } + await assertOwnerMarker(); + return await action(); + } finally { + await releaseGuard(); + } + }; + + const heartbeatTimer = setInterval( () => { + heartbeatPromise = heartbeatPromise.then( async () => { + try { + const releaseGuard = await this.waitForLoginLockGuard(); + try { + await assertOwnerMarker(); + const now = new Date(); + await fs.utimes( lockPath, now, now ); + } finally { + await releaseGuard(); + } + } catch ( error ) { + heartbeatError ??= error; + } + } ); + }, LOCK_HEARTBEAT_MS ); + heartbeatTimer.unref(); + + return { + assertOwned: () => runIfOwned( async () => {} ), + runIfOwned, + release: async () => { + clearInterval( heartbeatTimer ); + await heartbeatPromise; + await this.releaseLoginLock( ownerPath ); + }, + }; + } + + /** Releases this worker's lock without touching a replacement owner's lock. */ + private async releaseLoginLock( ownerPath: string ): Promise< void > { + const releaseGuard = await this.waitForLoginLockGuard(); + try { + try { + await fs.unlink( ownerPath ); + } catch ( error ) { + if ( hasErrorCode( error, 'ENOENT' ) ) { + return; + } + throw error; + } + + try { + await fs.rmdir( this.getLoginLockPath() ); + } catch ( error ) { + if ( ! hasErrorCode( error, 'ENOENT', 'ENOTEMPTY' ) ) { + throw error; + } + } + } finally { + await releaseGuard(); + } + } + + /** Waits briefly for the guard that serializes lock directory changes. */ + private async waitForLoginLockGuard(): Promise< () => Promise< void > > { + const deadline = Date.now() + LOCK_GUARD_WAIT_MS; + for (;;) { + const releaseGuard = await this.acquireLoginLockGuard(); + if ( releaseGuard ) { + return releaseGuard; + } + if ( Date.now() >= deadline ) { + throw new Error( `Timed out waiting for login lock guard for ${ this.accountName }` ); + } + await new Promise( ( resolve ) => setTimeout( resolve, 50 ) ); + } + } + + /** Tries to acquire the guard that serializes lock directory changes. */ + private async acquireLoginLockGuard(): Promise< ( () => Promise< void > ) | null > { + const guardPath = `${ this.getLoginLockPath() }.guard`; + const ownerPath = path.join( guardPath, `${ randomUUID() }.owner` ); + + try { + await fs.mkdir( guardPath ); + } catch ( error ) { + if ( ! hasErrorCode( error, 'EEXIST' ) ) { + throw error; + } + + let newestMtime: number; + let owners: string[]; + try { + newestMtime = ( await fs.stat( guardPath ) ).mtimeMs; + owners = await fs.readdir( guardPath ); + } catch ( inspectError ) { + if ( hasErrorCode( inspectError, 'ENOENT' ) ) { + return null; + } + throw inspectError; + } + + for ( const owner of owners ) { + try { + newestMtime = Math.max( + newestMtime, + ( await fs.stat( path.join( guardPath, owner ) ) ).mtimeMs + ); + } catch ( inspectError ) { + if ( hasErrorCode( inspectError, 'ENOENT' ) ) { + return null; + } + throw inspectError; + } + } + + if ( Date.now() < newestMtime + LOCK_GUARD_STALE_MS ) { + return null; + } + + const stalePath = `${ guardPath }.${ randomUUID() }.stale`; + try { + await fs.rename( guardPath, stalePath ); + } catch ( renameError ) { + if ( hasErrorCode( renameError, 'ENOENT' ) ) { + return null; + } + throw renameError; + } + await fs.rm( stalePath, { force: true, recursive: true } ); + return null; + } + + try { + await fs.mkdir( ownerPath ); + } catch ( error ) { + await fs.rmdir( guardPath ).catch( () => {} ); + throw error; + } + + let heartbeatError: unknown; + let heartbeatPromise = Promise.resolve(); + const heartbeatTimer = setInterval( () => { + heartbeatPromise = heartbeatPromise.then( async () => { + try { + const now = new Date(); + await fs.utimes( ownerPath, now, now ); + } catch ( error ) { + heartbeatError ??= error; + } + } ); + }, LOCK_GUARD_HEARTBEAT_MS ); + heartbeatTimer.unref(); + + return async () => { + clearInterval( heartbeatTimer ); + await heartbeatPromise; + if ( heartbeatError ) { + throw new Error( `Lost login lock guard for ${ this.accountName }`, { + cause: heartbeatError, + } ); + } + try { + await fs.rmdir( ownerPath ); + } catch ( error ) { + if ( hasErrorCode( error, 'ENOENT' ) ) { + throw new Error( `Lost login lock guard for ${ this.accountName }`, { cause: error } ); + } + throw error; + } + await fs.rmdir( guardPath ); + }; + } + + /** Returns this account's login lock directory. */ + private getLoginLockPath(): string { + return path.join( envVariables.COOKIES_PATH, `${ this.accountName }.lock` ); + } + /** * Logs in via the login page UI. The verification code will be submitted * automatically if it's defined in the secrets file. @@ -189,8 +569,11 @@ export class TestAccount { const twoDaysMs = 2 * 24 * 60 * 60 * 1000; return nowMs < birthtimeMs + twoDaysMs; - } catch { - return false; + } catch ( error ) { + if ( hasErrorCode( error, 'ENOENT' ) ) { + return false; + } + throw error; } } diff --git a/packages/calypso-e2e/src/test/test-account.login-lock.test.ts b/packages/calypso-e2e/src/test/test-account.login-lock.test.ts new file mode 100644 index 000000000000..88b9cf664bec --- /dev/null +++ b/packages/calypso-e2e/src/test/test-account.login-lock.test.ts @@ -0,0 +1,290 @@ +import { + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + rmSync, + utimesSync, + writeFileSync, +} from 'node:fs'; +import fs from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { afterAll, afterEach, beforeEach, describe, expect, jest, test } from '@jest/globals'; +import { TestAccount } from '../lib/test-account'; +import { SecretsManager } from '../secrets'; +import type { Secrets } from '../secrets'; +import type { BrowserContext, Page } from 'playwright'; + +jest.spyOn( SecretsManager, 'secrets', 'get' ).mockImplementation( + () => + ( { + testAccounts: { defaultUser: { username: 'user', password: 'pass' } }, + } ) as unknown as Secrets +); + +const AMBIENT_COOKIES_PATH = process.env.COOKIES_PATH; + +let cookiesPath: string; + +beforeEach( () => { + cookiesPath = mkdtempSync( path.join( tmpdir(), 'login-lock-' ) ); + process.env.COOKIES_PATH = cookiesPath; +} ); + +afterEach( () => { + rmSync( cookiesPath, { force: true, recursive: true } ); +} ); + +// Jest workers are reused across test files, so hand the environment back. +afterAll( () => { + if ( AMBIENT_COOKIES_PATH === undefined ) { + delete process.env.COOKIES_PATH; + } else { + process.env.COOKIES_PATH = AMBIENT_COOKIES_PATH; + } +} ); + +/** + * Builds an account whose login writes the cookies file the way the real one does, and + * counts how many times it was called. + * + * @param {number} durationMs How long the fake login takes. + */ +function accountWithCountedLogin( durationMs = 50 ) { + const account = new TestAccount( 'defaultUser' ); + const calls = { count: 0 }; + const clearCookies = jest.fn(); + const context = { clearCookies } as unknown as BrowserContext; + + jest.spyOn( account, 'logInViaLoginPage' ).mockImplementation( async () => { + expect( clearCookies ).toHaveBeenCalledTimes( 1 ); + calls.count++; + await new Promise( ( resolve ) => setTimeout( resolve, durationMs ) ); + } ); + jest.spyOn( account, 'saveAuthCookies' ).mockImplementation( async () => { + writeFileSync( path.join( cookiesPath, 'defaultUser.json' ), '{"cookies":[]}' ); + } ); + + const page = { context: () => context } as Page; + + return { account, calls, clearCookies, page }; +} + +/** Creates a promise controlled by the test. */ +function deferred() { + let resolve!: () => void; + const promise = new Promise< void >( ( resolvePromise ) => { + resolve = resolvePromise; + } ); + return { promise, resolve }; +} + +/** Waits for an asynchronous test condition. */ +async function waitFor( condition: () => boolean ): Promise< void > { + const deadline = Date.now() + 5 * 1000; + while ( ! condition() ) { + if ( Date.now() >= deadline ) { + throw new Error( 'Timed out waiting for test condition' ); + } + await new Promise( ( resolve ) => setTimeout( resolve, 10 ) ); + } +} + +describe( 'TestAccount login lock', () => { + test( 'concurrent workers log in once between them', async () => { + const workers = Array.from( { length: 8 }, () => accountWithCountedLogin() ); + + const loggedIn = await Promise.all( + workers.map( ( { account, page } ) => account.ensureFreshAuthCookies( page ) ) + ); + + expect( workers.reduce( ( total, { calls } ) => total + calls.count, 0 ) ).toBe( 1 ); + // Exactly one worker reports having logged in; the rest waited for its cookies. + expect( loggedIn.filter( Boolean ) ).toHaveLength( 1 ); + expect( existsSync( path.join( cookiesPath, 'defaultUser.json' ) ) ).toBe( true ); + } ); + + test( 'the lock is released once the login is done', async () => { + const { account, page } = accountWithCountedLogin(); + + await account.ensureFreshAuthCookies( page ); + + expect( existsSync( path.join( cookiesPath, 'defaultUser.lock' ) ) ).toBe( false ); + } ); + + test( 'the lock is released when the login throws', async () => { + const account = new TestAccount( 'defaultUser' ); + const context = { clearCookies: jest.fn() } as unknown as BrowserContext; + const page = { context: () => context } as Page; + jest.spyOn( account, 'logInViaLoginPage' ).mockRejectedValue( new Error( 'login blew up' ) ); + + await expect( account.ensureFreshAuthCookies( page ) ).rejects.toThrow( 'login blew up' ); + expect( existsSync( path.join( cookiesPath, 'defaultUser.lock' ) ) ).toBe( false ); + } ); + + test( 'a release failure preserves the login error', async () => { + const account = new TestAccount( 'defaultUser' ); + const context = { clearCookies: jest.fn() } as unknown as BrowserContext; + const page = { context: () => context } as Page; + const loginError = new Error( 'login blew up' ); + const releaseError = Object.assign( new Error( 'release blew up' ), { code: 'EACCES' } ); + jest.spyOn( account, 'logInViaLoginPage' ).mockRejectedValue( loginError ); + const unlink = jest.spyOn( fs, 'unlink' ).mockRejectedValueOnce( releaseError ); + + try { + const result = account.ensureFreshAuthCookies( page ); + await expect( result ).rejects.toEqual( + expect.objectContaining( { + errors: [ loginError, releaseError ], + } ) + ); + } finally { + unlink.mockRestore(); + } + } ); + + test( 'a transient login failure is retried by the lock owner', async () => { + const account = new TestAccount( 'defaultUser' ); + const context = { clearCookies: jest.fn() } as unknown as BrowserContext; + const page = { context: () => context } as Page; + const login = jest + .spyOn( account, 'logInViaLoginPage' ) + .mockRejectedValueOnce( new Error( 'navigation timed out' ) ) + .mockResolvedValueOnce(); + jest.spyOn( account, 'saveAuthCookies' ).mockImplementation( async () => { + writeFileSync( path.join( cookiesPath, 'defaultUser.json' ), '{"cookies":[]}' ); + } ); + + await expect( account.ensureFreshAuthCookies( page ) ).resolves.toBe( true ); + expect( login ).toHaveBeenCalledTimes( 2 ); + expect( context.clearCookies ).toHaveBeenCalledTimes( 2 ); + } ); + + test( 'a lock left behind by a dead worker is taken over', async () => { + const lockPath = path.join( cookiesPath, 'defaultUser.lock' ); + mkdirSync( lockPath ); + writeFileSync( path.join( lockPath, 'dead-worker.owner' ), '' ); + const longAgo = new Date( Date.now() - 10 * 60 * 1000 ); + utimesSync( lockPath, longAgo, longAgo ); + + const { account, calls, page } = accountWithCountedLogin(); + await account.ensureFreshAuthCookies( page ); + + expect( calls.count ).toBe( 1 ); + // The stolen lock is renamed out of the way and removed, not left to accumulate. + expect( readdirSync( cookiesPath ) ).toEqual( [ 'defaultUser.json' ] ); + } ); + + test( 'a guard left behind by a dead worker is taken over', async () => { + const guardPath = path.join( cookiesPath, 'defaultUser.lock.guard' ); + mkdirSync( guardPath ); + const longAgo = new Date( Date.now() - 10 * 60 * 1000 ); + utimesSync( guardPath, longAgo, longAgo ); + + const { account, calls, page } = accountWithCountedLogin(); + await account.ensureFreshAuthCookies( page ); + + expect( calls.count ).toBe( 1 ); + expect( readdirSync( cookiesPath ) ).toEqual( [ 'defaultUser.json' ] ); + } ); + + test( 'a stale owner cannot release its replacement lock', async () => { + const firstLogin = deferred(); + const secondLogin = deferred(); + const secondLoginStarted = deferred(); + const first = new TestAccount( 'defaultUser' ); + const second = new TestAccount( 'defaultUser' ); + const firstContext = { clearCookies: jest.fn() } as unknown as BrowserContext; + const secondContext = { clearCookies: jest.fn() } as unknown as BrowserContext; + const firstPage = { context: () => firstContext } as Page; + const secondPage = { context: () => secondContext } as Page; + const firstSave = jest.spyOn( first, 'saveAuthCookies' ).mockResolvedValue(); + + jest.spyOn( first, 'logInViaLoginPage' ).mockReturnValue( firstLogin.promise ); + jest.spyOn( second, 'logInViaLoginPage' ).mockImplementation( async () => { + secondLoginStarted.resolve(); + await secondLogin.promise; + } ); + jest.spyOn( second, 'saveAuthCookies' ).mockImplementation( async () => { + writeFileSync( path.join( cookiesPath, 'defaultUser.json' ), '{"cookies":[]}' ); + } ); + + const lockPath = path.join( cookiesPath, 'defaultUser.lock' ); + const firstResult = first.ensureFreshAuthCookies( firstPage ); + const firstRejection = expect( firstResult ).rejects.toThrow( 'Lost login lock' ); + await waitFor( () => existsSync( lockPath ) ); + const longAgo = new Date( Date.now() - 10 * 60 * 1000 ); + utimesSync( lockPath, longAgo, longAgo ); + + const secondResult = second.ensureFreshAuthCookies( secondPage ); + await secondLoginStarted.promise; + firstLogin.resolve(); + await firstRejection; + + expect( firstSave ).not.toHaveBeenCalled(); + expect( existsSync( lockPath ) ).toBe( true ); + expect( readdirSync( lockPath ) ).toHaveLength( 1 ); + + secondLogin.resolve(); + await expect( secondResult ).resolves.toBe( true ); + } ); + + test( 'a failed ownership marker does not leave a lock behind', async () => { + const error = Object.assign( new Error( 'permission denied' ), { code: 'EACCES' } ); + const writeFile = jest.spyOn( fs, 'writeFile' ).mockRejectedValueOnce( error ); + const { account, calls, page } = accountWithCountedLogin(); + + try { + await expect( account.ensureFreshAuthCookies( page ) ).rejects.toBe( error ); + expect( calls.count ).toBe( 0 ); + expect( existsSync( path.join( cookiesPath, 'defaultUser.lock' ) ) ).toBe( false ); + } finally { + writeFile.mockRestore(); + } + } ); + + test( 'unexpected freshness errors are propagated', async () => { + const account = new TestAccount( 'defaultUser' ); + const error = Object.assign( new Error( 'permission denied' ), { code: 'EACCES' } ); + const stat = jest.spyOn( fs, 'stat' ).mockRejectedValueOnce( error ); + + try { + await expect( account.hasFreshAuthCookies() ).rejects.toBe( error ); + } finally { + stat.mockRestore(); + } + } ); + + test( 'unexpected stale-lock errors are propagated', async () => { + const lockPath = path.join( cookiesPath, 'defaultUser.lock' ); + mkdirSync( lockPath ); + writeFileSync( path.join( lockPath, 'dead-worker.owner' ), '' ); + const longAgo = new Date( Date.now() - 10 * 60 * 1000 ); + utimesSync( lockPath, longAgo, longAgo ); + const error = Object.assign( new Error( 'permission denied' ), { code: 'EACCES' } ); + const rename = jest.spyOn( fs, 'rename' ).mockRejectedValueOnce( error ); + const { account, calls, page } = accountWithCountedLogin(); + + try { + await expect( account.ensureFreshAuthCookies( page ) ).rejects.toBe( error ); + expect( calls.count ).toBe( 0 ); + } finally { + rename.mockRestore(); + } + } ); + + test( 'an expired cookies file is not mistaken for a fresh one', async () => { + // The freshness window is a birthtime the test cannot backdate, so stand in for an + // expired file. What is under test is that the wait ends on freshness rather than on + // the file merely being there. + writeFileSync( path.join( cookiesPath, 'defaultUser.json' ), '{"cookies":[]}' ); + + const { account, calls, page } = accountWithCountedLogin(); + jest.spyOn( account, 'hasFreshAuthCookies' ).mockResolvedValue( false as never ); + + await account.ensureFreshAuthCookies( page ); + + expect( calls.count ).toBe( 1 ); + } ); +} ); diff --git a/packages/calypso-e2e/src/types/env-variables.types.ts b/packages/calypso-e2e/src/types/env-variables.types.ts index 9aeebd02d373..025c637d588a 100644 --- a/packages/calypso-e2e/src/types/env-variables.types.ts +++ b/packages/calypso-e2e/src/types/env-variables.types.ts @@ -1,5 +1,3 @@ -import { TestAccountName } from '../'; - export type ViewportName = string; export type TestLocales = string[]; export type JetpackTarget = 'remote-site' | 'wpcom-production' | 'wpcom-deployment'; @@ -18,7 +16,6 @@ export interface SupportedEnvVariables { readonly ALLURE_RESULTS_PATH: string; readonly ARTIFACTS_PATH: string; readonly ATOMIC_VARIATION: AtomicVariation; - readonly AUTHENTICATE_ACCOUNTS: TestAccountName[]; readonly BROWSER_NAME: string; readonly CALYPSO_BASE_URL: string; readonly COBLOCKS_EDGE: boolean; diff --git a/test/e2e/AGENTS.md b/test/e2e/AGENTS.md index ee37ae120b14..7f3dcc7c7da1 100644 --- a/test/e2e/AGENTS.md +++ b/test/e2e/AGENTS.md @@ -141,7 +141,7 @@ test( 'Test', async ( { pageLogin, componentSidebar } ) => { ### Available Fixtures -**Accounts**: one fixture per key of `fixtureAccounts` in [`lib/pw-base.ts`](lib/pw-base.ts), plus `accountGivenByEnvironment` and `accountSMS`. Adding a key there adds the fixture and primes the account before the suite. +**Accounts**: one fixture per key of `fixtureAccounts` in [`lib/pw-base.ts`](lib/pw-base.ts), plus `accountGivenByEnvironment` and `accountSMS`. Adding a key there adds the fixture; the account is logged in the first time a spec asks for it. **Pages/Components**: Follow naming conventions: diff --git a/test/e2e/docs/environment_variables.md b/test/e2e/docs/environment_variables.md index 2c901fc85a05..ae7860607b8b 100644 --- a/test/e2e/docs/environment_variables.md +++ b/test/e2e/docs/environment_variables.md @@ -9,7 +9,6 @@ Environment Variables control much of the runtime configuration for E2E tests. | Name | Description | Default | Required | | --------------------- | --------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------- | -------- | | ARTIFACTS_PATH | Path on disk to test artifacts (screenshots, logs, etc). | `./results/` | Optional | -| AUTHENTICATE_ACCOUNTS | Comma-delimited list of accounts to log in as before the suite runs. An empty value primes no account at all. | the list in `setup/prime-logins.setup.ts` | Optional | | CALYPSO_BASE_URL | The base URL to use for Calypso e.g. `https://wordpress.com`, `http://calypso.localhost:3000`, etc. | `http://calypso.localhost:3000` | Optional | | COBLOCKS_EDGE | Use the bleeding edge CoBlocks build. | `false` | Optional | | COOKIES_PATH | Path on disk to the saved authenticated cookies. | `./cookies/` | Optional | diff --git a/test/e2e/docs/tests_local.md b/test/e2e/docs/tests_local.md index f6d7e4b373f6..b5f346edcb0b 100644 --- a/test/e2e/docs/tests_local.md +++ b/test/e2e/docs/tests_local.md @@ -70,17 +70,11 @@ See the [list of groups](tests_ci.md#featuretest-groups). ### Save authentication cookies -The `prime-logins` setup project logs in as a list of accounts before the main test suite runs and saves their cookies to be re-used until expiry (typically 3 days). Every project except `authentication` waits for it, so the specs read those cookies instead of logging in themselves. +The first test that needs an account logs in and saves its cookies under `COOKIES_PATH`, to be re-used until expiry (typically 2 days). Every other test and worker reads that file instead of logging in again. -By default it primes the list in [`setup/prime-logins.setup.ts`](../setup/prime-logins.setup.ts). To prime a different set, name the accounts found in [Secret Manager](../../../packages/calypso-e2e/src/secrets/secrets-manager.ts), separated by commas: +Workers that need the same account at the same time race for a lock beside the cookies: the winner logs in, the rest wait for the cookies it writes. So each account is logged in once per run, whichever spec gets there first, and nothing has to be listed anywhere in advance. -```bash -export AUTHENTICATE_ACCOUNTS=simpleSitePersonalPlanUser,atomicUser,defaultUser -``` - -Either list is primed alongside the account the current environment variables select, the one behind the `accountGivenByEnvironment` fixture. - -Set it to an empty value to skip priming altogether; whatever needs an account then logs in when it first runs. +Delete the `cookies` directory to force a fresh login. ### Use the mobile viewport diff --git a/test/e2e/lib/get-account.ts b/test/e2e/lib/get-account.ts index 6cfc4cf3dc3e..a03cf344648d 100644 --- a/test/e2e/lib/get-account.ts +++ b/test/e2e/lib/get-account.ts @@ -2,10 +2,10 @@ import { TestAccount, TestAccountName } from '@automattic/calypso-e2e'; import { Page } from 'playwright'; /** - * Retrieves a `TestAccount` instance for the specified account name, ensuring it has fresh authentication cookies. + * Retrieves and authenticates a `TestAccount` instance for the specified account name. * - * If the account does not have fresh authentication cookies, this function will log in via the login page - * and save the new authentication cookies to the browser context. + * If the account does not have fresh authentication cookies, this function logs in via the login page. + * Otherwise, it loads the saved cookies into the browser context. * * @param {Page} page - The Playwright `Page` instance to use for authentication actions. * @param {TestAccountName} accountName - The name of the test account to retrieve. @@ -13,19 +13,9 @@ import { Page } from 'playwright'; */ export async function getAccount( page: Page, - accountName: TestAccountName, - { isPriming = false }: { isPriming?: boolean } = {} + accountName: TestAccountName ): Promise< TestAccount > { const testAccount = new TestAccount( accountName ); - if ( ! ( await testAccount.hasFreshAuthCookies() ) ) { - if ( ! isPriming ) { - // The prime-logins setup project should have left cookies for this account. Say so - // when it didn't: a quiet run means priming worked, and a noisy one names the - // accounts that fell back to logging in one worker at a time. - console.log( `Logging in as ${ accountName }, no primed cookies to reuse.` ); - } - await testAccount.logInViaLoginPage( page ); - await testAccount.saveAuthCookies( page.context() ); - } + await testAccount.authenticate( page ); return testAccount; } diff --git a/test/e2e/lib/pw-base.ts b/test/e2e/lib/pw-base.ts index 74d215d8e4f2..786be459a515 100644 --- a/test/e2e/lib/pw-base.ts +++ b/test/e2e/lib/pw-base.ts @@ -111,10 +111,9 @@ export type CustomOptions = { /** * Test accounts exposed as a fixture of the same name, logged in on first use. * - * The `prime-logins` setup project logs in as each of these before the suite starts, so - * an account added here is primed rather than logged in inline. Two accounts are fixtures - * without belonging here: `accountGivenByEnvironment`, which resolves at run time, and - * `accountSMS`, whose 2FA code costs a Mailosaur email only a couple of specs need. + * Two accounts are fixtures without belonging here: `accountGivenByEnvironment`, which + * resolves at run time, and `accountSMS`, whose 2FA code costs a Mailosaur email only a + * couple of specs need. */ export const fixtureAccounts = { accountAtomic: 'atomicUser', @@ -682,11 +681,7 @@ export const test = base.extend< } }, sitePublicShared: async ( { page, helperData }, use ) => { - // getAccount persists auth cookies on first login so parallel tests reuse - // them instead of each re-logging-in; authenticate then loads them onto - // this test's page (needed by the import navigation). const account = await getAccount( page, 'defaultUser' ); - await account.authenticate( page ); // createSite is the first line that creates a real resource. From here on // everything is wrapped so the site is deleted no matter what happens next: diff --git a/test/e2e/playwright.config.ts b/test/e2e/playwright.config.ts index e8d9ac6aa358..9fd2423c0b6a 100644 --- a/test/e2e/playwright.config.ts +++ b/test/e2e/playwright.config.ts @@ -110,21 +110,9 @@ export default defineConfig( { testMatch: /mailosaur-usage\.setup\.ts/, testDir: './setup', }, - { - name: 'prime-logins', - testMatch: /prime-logins\.setup\.ts/, - testDir: './setup', - // Borrows the `chrome` context so the login carries the e2e user agent suffix the - // backend expects. The cookies it leaves are per account, not per device, so the - // mobile projects reuse them too. - use: withCustomOptions( { - ...devices[ 'Desktop Chrome HiDPI' ], - userAgent: appendE2EUserAgent( devices[ 'Desktop Chrome HiDPI' ].userAgent ), - } ), - }, { name: 'chrome', - dependencies: [ 'mailosaur-usage-check', 'prime-logins' ], + dependencies: [ 'mailosaur-usage-check' ], use: withCustomOptions( { ...devices[ 'Desktop Chrome HiDPI' ], userAgent: appendE2EUserAgent( devices[ 'Desktop Chrome HiDPI' ].userAgent ), @@ -133,7 +121,7 @@ export default defineConfig( { }, { name: 'firefox', - dependencies: [ 'mailosaur-usage-check', 'prime-logins' ], + dependencies: [ 'mailosaur-usage-check' ], use: withCustomOptions( { ...devices[ 'Desktop Firefox' ], userAgent: appendE2EUserAgent( devices[ 'Desktop Firefox' ].userAgent ), @@ -142,7 +130,7 @@ export default defineConfig( { }, { name: 'webkit', - dependencies: [ 'mailosaur-usage-check', 'prime-logins' ], + dependencies: [ 'mailosaur-usage-check' ], use: withCustomOptions( { ...devices[ 'Desktop Safari' ], userAgent: appendE2EUserAgent( devices[ 'Desktop Safari' ].userAgent ), @@ -151,7 +139,7 @@ export default defineConfig( { }, { name: 'pixel', - dependencies: [ 'mailosaur-usage-check', 'prime-logins' ], + dependencies: [ 'mailosaur-usage-check' ], use: withCustomOptions( { ...devices[ 'Pixel 7' ], userAgent: appendE2EUserAgent( devices[ 'Pixel 7' ].userAgent ), @@ -161,7 +149,7 @@ export default defineConfig( { }, { name: 'galaxy', - dependencies: [ 'mailosaur-usage-check', 'prime-logins' ], + dependencies: [ 'mailosaur-usage-check' ], use: withCustomOptions( { ...devices[ 'Galaxy S24' ], userAgent: appendE2EUserAgent( devices[ 'Galaxy S24' ].userAgent ), @@ -171,7 +159,7 @@ export default defineConfig( { }, { name: 'iphone', - dependencies: [ 'mailosaur-usage-check', 'prime-logins' ], + dependencies: [ 'mailosaur-usage-check' ], use: withCustomOptions( { ...devices[ 'iPhone 15 Pro' ], userAgent: appendE2EUserAgent( devices[ 'iPhone 15 Pro' ].userAgent ), @@ -181,8 +169,6 @@ export default defineConfig( { }, { name: 'authentication', - // No 'prime-logins': these specs exercise the login flow itself, so warming - // the cookie cache would only add wall clock. dependencies: [ 'mailosaur-usage-check' ], retries: 0, testDir: './specs/authentication', diff --git a/test/e2e/setup/prime-logins.setup.ts b/test/e2e/setup/prime-logins.setup.ts deleted file mode 100644 index 05bd88d2a2d0..000000000000 --- a/test/e2e/setup/prime-logins.setup.ts +++ /dev/null @@ -1,111 +0,0 @@ -import { - envToFeatureKey, - envVariables, - getTestAccountByFeature, - type TestAccountName, -} from '@automattic/calypso-e2e'; -import { getAccount } from '../lib/get-account'; -import { fixtureAccounts, test as setup } from '../lib/pw-base'; - -// Accounts logged in as before the suite starts, so the specs read cookies instead of all -// logging in at once. -// -// Every worker begins by checking the shared cookie cache in COOKIES_PATH, which the -// TeamCity checkout wipes before each build. Without this project they all miss at once and -// log in through the UI concurrently, against a calypso.live container that has just been -// created, which is where most of the CI login timeouts come from. -// -// The account fixtures own this list, so adding a fixture primes it. An account missing from -// it still works: getAccount falls back to logging in inline, which is what every account did -// before this project existed. -const defaultAccountNames: TestAccountName[] = [ - ...Object.values( fixtureAccounts ), - // Not a fixture, but the account many specs select through a criteria override. - 'simpleSitePersonalPlanUser', -]; - -/** - * Returns the accounts to log in as before the suite starts. - * - * A build type that runs a narrow group can list just the accounts it needs in - * AUTHENTICATE_ACCOUNTS, or opt out of priming altogether by setting it to an empty value; - * the ToS build does the latter. - * - * Whichever list is used, the account this environment resolves `accountGivenByEnvironment` - * to is added to it: the Gutenberg edge, nightly, CoBlocks and Atomic builds each run - * against a different one, and it is the busiest account of those runs. It comes from a - * static table, so resolving it here costs nothing. - */ -function getAccountNamesToPrime(): TestAccountName[] { - let accountNames = defaultAccountNames; - - // Read process.env rather than the envVariables getter: the getter returns an empty array - // both when the variable is unset and when it is set to an empty value, and those mean - // different things here. - if ( process.env.AUTHENTICATE_ACCOUNTS !== undefined ) { - try { - accountNames = envVariables.AUTHENTICATE_ACCOUNTS; - } catch ( error ) { - // An unknown account name throws. This runs while the file is being collected, so - // letting it escape would fail the run before a single spec starts. - console.warn( `Ignoring AUTHENTICATE_ACCOUNTS: ${ error }` ); - } - } - - // An empty AUTHENTICATE_ACCOUNTS asks for no priming at all, so don't add back to it. - if ( accountNames.length === 0 ) { - return []; - } - - try { - return [ ...accountNames, getTestAccountByFeature( envToFeatureKey( envVariables ) ) ]; - } catch { - // No account is mapped to this environment; whatever needs one logs in inline. - return accountNames; - } -} - -// Well under the 120s test timeout. A login takes about 5s, so this only trips when -// something is badly wrong, and it leaves room for the retry to still finish in time. -const PRIME_TIMEOUT = 30 * 1000; - -for ( const accountName of new Set( getAccountNamesToPrime() ) ) { - setup( `prime login cookies: ${ accountName }`, async ( { page } ) => { - let timer: NodeJS.Timeout | undefined; - try { - // Race our own deadline: Playwright's test timeout aborts from the outside and - // can't be caught here, and a setup project that ends with a failing test makes - // Playwright skip every project that depends on it. That would cost the whole - // run instead of one account's inline login. - const priming = getAccount( page, accountName, { isPriming: true } ); - // When the deadline wins, the login keeps running until teardown closes the page - // and then rejects. Playwright charges an unhandled rejection to the test, which - // is the failure the deadline exists to avoid, so keep a handler on it. - priming.catch( () => {} ); - await Promise.race( [ - priming, - new Promise( ( _resolve, reject ) => { - timer = setTimeout( - () => reject( new Error( `timed out after ${ PRIME_TIMEOUT }ms` ) ), - PRIME_TIMEOUT - ); - } ), - ] ); - } catch ( error ) { - const info = setup.info(); - // Retrying is worth it, a stuck login usually succeeds second time round. Only - // the last attempt has to pass, so the project stays green either way. - if ( info.retry < info.project.retries ) { - throw error; - } - // Annotate rather than only log: the last attempt passes, so the reports are the - // only place a reader would otherwise see nothing at all. - info.annotations.push( { - type: 'prime-failed', - description: `Could not prime login cookies for ${ accountName }: ${ error }`, - } ); - } finally { - clearTimeout( timer ); - } - } ); -}