diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ed32496a49..bbc8584563 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -303,7 +303,9 @@ unit-bs: - .base-configuration - .bs-allowed-branches interruptible: true - resource_group: browserstack + parallel: + matrix: + - BS_BROWSER: [edge, firefox, safari-desktop, chrome-desktop, chrome-mobile] artifacts: reports: junit: test-report/unit-bs/*.xml diff --git a/scripts/test/bs-wrapper.ts b/scripts/test/bs-wrapper.ts index 58e5d860ee..32578649c4 100644 --- a/scripts/test/bs-wrapper.ts +++ b/scripts/test/bs-wrapper.ts @@ -1,7 +1,7 @@ -// This wrapper script ensures that no other test is running in BrowserStack before launching the +// This wrapper script waits until a BrowserStack parallel session is available before launching the // test command, to avoid overloading the service and making tests more flaky than necessary. This -// is also handled by the CI (exclusive lock on the "browserstack" resource), but it is helpful when -// launching tests outside of the CI. +// is also handled by the CI (resource groups), but it is helpful when launching tests outside of +// the CI. // // It used to re-run the test command based on its output (in particular, when the BrowserStack // session failed to be created), but we observed that: @@ -13,6 +13,7 @@ // after killing it. There might be a better way of prematurely aborting the test command if we need // to in the future. +import { randomUUID } from 'node:crypto' import { spawn, type ChildProcess } from 'node:child_process' import browserStack from 'browserstack-local' import { printLog, runMain, timeout, printError } from '../lib/executionUtils.ts' @@ -21,9 +22,10 @@ import { browserStackRequest } from '../lib/bsUtils.ts' const AVAILABILITY_CHECK_DELAY = 30_000 const NO_OUTPUT_TIMEOUT = 5 * 60_000 -const BS_BUILD_URL = 'https://api.browserstack.com/automate/builds.json?status=running' +const BS_PLAN_URL = 'https://api.browserstack.com/automate/plan.json' const bsLocal = new browserStack.Local() +const localIdentifier = `browser-sdk-${randomUUID()}` runMain(async () => { if (command`git tag --points-at HEAD`.run()) { @@ -44,15 +46,19 @@ runMain(async () => { }) async function waitForAvailability(): Promise { - while (await hasRunningBuild()) { - printLog('Other build running, waiting...') - await timeout(AVAILABILITY_CHECK_DELAY) + while (await isAtCapacity()) { + const jitter = Math.floor(Math.random() * 3_000) + printLog('All BrowserStack sessions occupied, waiting...') + await timeout(AVAILABILITY_CHECK_DELAY + jitter) } } -async function hasRunningBuild(): Promise { - const builds = (await browserStackRequest(BS_BUILD_URL)) as any[] - return builds.length > 0 +async function isAtCapacity(): Promise { + const plan = (await browserStackRequest(BS_PLAN_URL)) as { + parallel_sessions_running: number + parallel_sessions_max_allowed: number + } + return plan.parallel_sessions_running >= plan.parallel_sessions_max_allowed } function startBsLocal(): Promise { @@ -62,8 +68,8 @@ function startBsLocal(): Promise { bsLocal.start( { key: process.env.BS_ACCESS_KEY, + localIdentifier, forceLocal: true, - forceKill: true, onlyAutomate: true, }, (error?: Error) => { @@ -97,6 +103,7 @@ function runTests(): Promise { ...process.env, FORCE_COLOR: 'true', BROWSER_STACK: 'true', + BROWSERSTACK_LOCAL_IDENTIFIER: localIdentifier, }, }) diff --git a/test/browsers.conf.d.ts b/test/browsers.conf.d.ts index 231cc71575..1bc8f4189b 100644 --- a/test/browsers.conf.d.ts +++ b/test/browsers.conf.d.ts @@ -1,4 +1,5 @@ export interface BrowserConfiguration { + id?: string sessionName: string name: string version?: string diff --git a/test/unit/browsers.conf.ts b/test/unit/browsers.conf.ts index 11db782976..fd0bb06664 100644 --- a/test/unit/browsers.conf.ts +++ b/test/unit/browsers.conf.ts @@ -9,6 +9,7 @@ export const OLDEST_BROWSER_ECMA_VERSION = 2017 export const browserConfigurations: BrowserConfiguration[] = [ { + id: 'edge', sessionName: 'Edge', name: 'Edge', version: '80.0', @@ -16,6 +17,7 @@ export const browserConfigurations: BrowserConfiguration[] = [ osVersion: '11', }, { + id: 'firefox', sessionName: 'Firefox', name: 'Firefox', version: '78.0', @@ -23,6 +25,7 @@ export const browserConfigurations: BrowserConfiguration[] = [ osVersion: '11', }, { + id: 'safari-desktop', sessionName: 'Safari desktop', name: 'Safari', version: '14.0', @@ -30,6 +33,7 @@ export const browserConfigurations: BrowserConfiguration[] = [ osVersion: 'Big Sur', }, { + id: 'chrome-desktop', sessionName: 'Chrome desktop', name: 'Chrome', version: '80.0', @@ -37,6 +41,7 @@ export const browserConfigurations: BrowserConfiguration[] = [ osVersion: '11', }, { + id: 'chrome-mobile', sessionName: 'Chrome mobile', name: 'chrome', os: 'android', diff --git a/test/unit/karma.bs.conf.js b/test/unit/karma.bs.conf.js index c6b4314a8c..f7af94ddbe 100644 --- a/test/unit/karma.bs.conf.js +++ b/test/unit/karma.bs.conf.js @@ -2,6 +2,16 @@ import { getBuildInfos } from '../envUtils.ts' import { browserConfigurations } from './browsers.conf.ts' import karmaBaseConf from './karma.base.conf.js' +const selectedBrowser = process.env.BS_BROWSER +const filteredConfigurations = selectedBrowser + ? browserConfigurations.filter((configuration) => configuration.id === selectedBrowser) + : browserConfigurations + +if (selectedBrowser && filteredConfigurations.length === 0) { + const availableIds = browserConfigurations.map((c) => c.id).join(', ') + throw new Error(`Unknown BS_BROWSER "${selectedBrowser}". Available: ${availableIds}`) +} + // eslint-disable-next-line import/no-default-export export default function (config) { config.set({ @@ -13,19 +23,20 @@ export default function (config) { ], plugins: [...karmaBaseConf.plugins, 'karma-browserstack-launcher'], reporters: [...karmaBaseConf.reporters, 'BrowserStack'], - browsers: browserConfigurations.map((configuration) => configuration.sessionName), - concurrency: 5, + browsers: filteredConfigurations.map((configuration) => configuration.sessionName), + concurrency: filteredConfigurations.length, browserDisconnectTolerance: 3, captureTimeout: 2 * 60 * 1000, browserStack: { username: process.env.BS_USERNAME, accessKey: process.env.BS_ACCESS_KEY, + localIdentifier: process.env.BROWSERSTACK_LOCAL_IDENTIFIER, project: 'browser sdk unit', build: getBuildInfos(), video: false, }, customLaunchers: Object.fromEntries( - browserConfigurations.map((configuration) => [ + filteredConfigurations.map((configuration) => [ configuration.sessionName, // See https://github.com/karma-runner/karma-browserstack-launcher#per-browser-options {