From df95319b0546a4b8de06e91231e09cfa4dceef9a Mon Sep 17 00:00:00 2001 From: Adrian de la Rosa Date: Mon, 25 May 2026 17:05:25 +0200 Subject: [PATCH 1/3] =?UTF-8?q?=F0=9F=91=B7=20Split=20unit-bs=20CI=20job?= =?UTF-8?q?=20per=20browser?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each browser now runs as a separate CI job using parallel: matrix. The --browser flag on ci-bs.ts filters the Karma config to a single browser via the BS_BROWSER env var. --- .gitlab-ci.yml | 6 ++++-- scripts/test/ci-bs.ts | 33 ++++++++++++++++++++++++--------- test/browsers.conf.d.ts | 1 + test/unit/browsers.conf.ts | 5 +++++ test/unit/karma.bs.conf.js | 16 +++++++++++++--- 5 files changed, 47 insertions(+), 14 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ed32496a49..cea83108cd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -303,13 +303,15 @@ unit-bs: - .base-configuration - .bs-allowed-branches interruptible: true - resource_group: browserstack + parallel: + matrix: + - BS_BROWSER: [edge, firefox, safari-desktop, chrome-desktop, chrome-mobile] artifacts: reports: junit: test-report/unit-bs/*.xml script: - yarn - - node scripts/test/ci-bs.ts test:unit + - node scripts/test/ci-bs.ts test:unit --browser $BS_BROWSER after_script: - node ./scripts/test/export-test-result.ts unit-bs diff --git a/scripts/test/ci-bs.ts b/scripts/test/ci-bs.ts index 9b233c13dc..c3adba131c 100644 --- a/scripts/test/ci-bs.ts +++ b/scripts/test/ci-bs.ts @@ -1,3 +1,4 @@ +import { parseArgs } from 'node:util' import { printLog, runMain } from '../lib/executionUtils.ts' import { command } from '../lib/command.ts' import { fetchPR, getLastCommonCommit, LOCAL_BRANCH } from '../lib/gitUtils.ts' @@ -16,9 +17,20 @@ const RELEVANT_FILE_PATTERNS = [ ] runMain(async () => { - const testCommand = process.argv[2] + const { + values: { browser }, + positionals, + } = parseArgs({ + args: process.argv.slice(2), + options: { + browser: { type: 'string' }, + }, + allowPositionals: true, + }) + + const testCommand = positionals[0] if (!testCommand) { - throw new Error('Usage: ci-bs.ts ') + throw new Error('Usage: ci-bs.ts [--browser ]') } const pr = await fetchPR(LOCAL_BRANCH!) @@ -30,13 +42,16 @@ runMain(async () => { return } - command`yarn ${testCommand}:bs` - .withEnvironment({ - BS_USERNAME: getBrowserStackUsername(), - BS_ACCESS_KEY: getBrowserStackAccessKey(), - }) - .withLogs() - .run() + const environment: Record = { + BS_USERNAME: getBrowserStackUsername(), + BS_ACCESS_KEY: getBrowserStackAccessKey(), + } + + if (browser) { + environment.BS_BROWSER = browser + } + + command`yarn ${testCommand}:bs`.withEnvironment(environment).withLogs().run() }) function hasRelevantChanges(baseCommit: string): boolean { diff --git a/test/browsers.conf.d.ts b/test/browsers.conf.d.ts index 231cc71575..1bc8f4189b 100644 --- a/test/browsers.conf.d.ts +++ b/test/browsers.conf.d.ts @@ -1,4 +1,5 @@ export interface BrowserConfiguration { + id?: string sessionName: string name: string version?: string diff --git a/test/unit/browsers.conf.ts b/test/unit/browsers.conf.ts index 11db782976..fd0bb06664 100644 --- a/test/unit/browsers.conf.ts +++ b/test/unit/browsers.conf.ts @@ -9,6 +9,7 @@ export const OLDEST_BROWSER_ECMA_VERSION = 2017 export const browserConfigurations: BrowserConfiguration[] = [ { + id: 'edge', sessionName: 'Edge', name: 'Edge', version: '80.0', @@ -16,6 +17,7 @@ export const browserConfigurations: BrowserConfiguration[] = [ osVersion: '11', }, { + id: 'firefox', sessionName: 'Firefox', name: 'Firefox', version: '78.0', @@ -23,6 +25,7 @@ export const browserConfigurations: BrowserConfiguration[] = [ osVersion: '11', }, { + id: 'safari-desktop', sessionName: 'Safari desktop', name: 'Safari', version: '14.0', @@ -30,6 +33,7 @@ export const browserConfigurations: BrowserConfiguration[] = [ osVersion: 'Big Sur', }, { + id: 'chrome-desktop', sessionName: 'Chrome desktop', name: 'Chrome', version: '80.0', @@ -37,6 +41,7 @@ export const browserConfigurations: BrowserConfiguration[] = [ osVersion: '11', }, { + id: 'chrome-mobile', sessionName: 'Chrome mobile', name: 'chrome', os: 'android', diff --git a/test/unit/karma.bs.conf.js b/test/unit/karma.bs.conf.js index c6b4314a8c..28589d17da 100644 --- a/test/unit/karma.bs.conf.js +++ b/test/unit/karma.bs.conf.js @@ -2,6 +2,16 @@ import { getBuildInfos } from '../envUtils.ts' import { browserConfigurations } from './browsers.conf.ts' import karmaBaseConf from './karma.base.conf.js' +const selectedBrowser = process.env.BS_BROWSER +const filteredConfigurations = selectedBrowser + ? browserConfigurations.filter((configuration) => configuration.id === selectedBrowser) + : browserConfigurations + +if (selectedBrowser && filteredConfigurations.length === 0) { + const availableIds = browserConfigurations.map((c) => c.id).join(', ') + throw new Error(`Unknown BS_BROWSER "${selectedBrowser}". Available: ${availableIds}`) +} + // eslint-disable-next-line import/no-default-export export default function (config) { config.set({ @@ -13,8 +23,8 @@ export default function (config) { ], plugins: [...karmaBaseConf.plugins, 'karma-browserstack-launcher'], reporters: [...karmaBaseConf.reporters, 'BrowserStack'], - browsers: browserConfigurations.map((configuration) => configuration.sessionName), - concurrency: 5, + browsers: filteredConfigurations.map((configuration) => configuration.sessionName), + concurrency: filteredConfigurations.length, browserDisconnectTolerance: 3, captureTimeout: 2 * 60 * 1000, browserStack: { @@ -25,7 +35,7 @@ export default function (config) { video: false, }, customLaunchers: Object.fromEntries( - browserConfigurations.map((configuration) => [ + filteredConfigurations.map((configuration) => [ configuration.sessionName, // See https://github.com/karma-runner/karma-browserstack-launcher#per-browser-options { From 17b291f55b36870dfb0abdbba1b241345d3f0517 Mon Sep 17 00:00:00 2001 From: Adrian de la Rosa Date: Mon, 25 May 2026 17:06:00 +0200 Subject: [PATCH 2/3] =?UTF-8?q?=F0=9F=91=B7=20Improve=20bs-wrapper=20for?= =?UTF-8?q?=20parallel=20BrowserStack=20execution?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wait for available sessions instead of blocking on any running build. Isolate tunnels with a unique localIdentifier per run. Add jitter to the availability check to prevent thundering herd. --- scripts/test/bs-wrapper.ts | 29 ++++++++++++++++++----------- test/unit/karma.bs.conf.js | 1 + 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/scripts/test/bs-wrapper.ts b/scripts/test/bs-wrapper.ts index 58e5d860ee..32578649c4 100644 --- a/scripts/test/bs-wrapper.ts +++ b/scripts/test/bs-wrapper.ts @@ -1,7 +1,7 @@ -// This wrapper script ensures that no other test is running in BrowserStack before launching the +// This wrapper script waits until a BrowserStack parallel session is available before launching the // test command, to avoid overloading the service and making tests more flaky than necessary. This -// is also handled by the CI (exclusive lock on the "browserstack" resource), but it is helpful when -// launching tests outside of the CI. +// is also handled by the CI (resource groups), but it is helpful when launching tests outside of +// the CI. // // It used to re-run the test command based on its output (in particular, when the BrowserStack // session failed to be created), but we observed that: @@ -13,6 +13,7 @@ // after killing it. There might be a better way of prematurely aborting the test command if we need // to in the future. +import { randomUUID } from 'node:crypto' import { spawn, type ChildProcess } from 'node:child_process' import browserStack from 'browserstack-local' import { printLog, runMain, timeout, printError } from '../lib/executionUtils.ts' @@ -21,9 +22,10 @@ import { browserStackRequest } from '../lib/bsUtils.ts' const AVAILABILITY_CHECK_DELAY = 30_000 const NO_OUTPUT_TIMEOUT = 5 * 60_000 -const BS_BUILD_URL = 'https://api.browserstack.com/automate/builds.json?status=running' +const BS_PLAN_URL = 'https://api.browserstack.com/automate/plan.json' const bsLocal = new browserStack.Local() +const localIdentifier = `browser-sdk-${randomUUID()}` runMain(async () => { if (command`git tag --points-at HEAD`.run()) { @@ -44,15 +46,19 @@ runMain(async () => { }) async function waitForAvailability(): Promise { - while (await hasRunningBuild()) { - printLog('Other build running, waiting...') - await timeout(AVAILABILITY_CHECK_DELAY) + while (await isAtCapacity()) { + const jitter = Math.floor(Math.random() * 3_000) + printLog('All BrowserStack sessions occupied, waiting...') + await timeout(AVAILABILITY_CHECK_DELAY + jitter) } } -async function hasRunningBuild(): Promise { - const builds = (await browserStackRequest(BS_BUILD_URL)) as any[] - return builds.length > 0 +async function isAtCapacity(): Promise { + const plan = (await browserStackRequest(BS_PLAN_URL)) as { + parallel_sessions_running: number + parallel_sessions_max_allowed: number + } + return plan.parallel_sessions_running >= plan.parallel_sessions_max_allowed } function startBsLocal(): Promise { @@ -62,8 +68,8 @@ function startBsLocal(): Promise { bsLocal.start( { key: process.env.BS_ACCESS_KEY, + localIdentifier, forceLocal: true, - forceKill: true, onlyAutomate: true, }, (error?: Error) => { @@ -97,6 +103,7 @@ function runTests(): Promise { ...process.env, FORCE_COLOR: 'true', BROWSER_STACK: 'true', + BROWSERSTACK_LOCAL_IDENTIFIER: localIdentifier, }, }) diff --git a/test/unit/karma.bs.conf.js b/test/unit/karma.bs.conf.js index 28589d17da..f7af94ddbe 100644 --- a/test/unit/karma.bs.conf.js +++ b/test/unit/karma.bs.conf.js @@ -30,6 +30,7 @@ export default function (config) { browserStack: { username: process.env.BS_USERNAME, accessKey: process.env.BS_ACCESS_KEY, + localIdentifier: process.env.BROWSERSTACK_LOCAL_IDENTIFIER, project: 'browser sdk unit', build: getBuildInfos(), video: false, From 84a9b8741ebee40ad5a67e2e35e07a2f1f97004f Mon Sep 17 00:00:00 2001 From: Adrian de la Rosa Date: Tue, 26 May 2026 11:29:53 +0200 Subject: [PATCH 3/3] =?UTF-8?q?=F0=9F=91=B7=20Drop=20--browser=20flag,=20r?= =?UTF-8?q?ely=20on=20BS=5FBROWSER=20env=20var=20from=20matrix?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitlab-ci.yml | 2 +- scripts/test/ci-bs.ts | 33 +++++++++------------------------ 2 files changed, 10 insertions(+), 25 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cea83108cd..bbc8584563 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -311,7 +311,7 @@ unit-bs: junit: test-report/unit-bs/*.xml script: - yarn - - node scripts/test/ci-bs.ts test:unit --browser $BS_BROWSER + - node scripts/test/ci-bs.ts test:unit after_script: - node ./scripts/test/export-test-result.ts unit-bs diff --git a/scripts/test/ci-bs.ts b/scripts/test/ci-bs.ts index c3adba131c..9b233c13dc 100644 --- a/scripts/test/ci-bs.ts +++ b/scripts/test/ci-bs.ts @@ -1,4 +1,3 @@ -import { parseArgs } from 'node:util' import { printLog, runMain } from '../lib/executionUtils.ts' import { command } from '../lib/command.ts' import { fetchPR, getLastCommonCommit, LOCAL_BRANCH } from '../lib/gitUtils.ts' @@ -17,20 +16,9 @@ const RELEVANT_FILE_PATTERNS = [ ] runMain(async () => { - const { - values: { browser }, - positionals, - } = parseArgs({ - args: process.argv.slice(2), - options: { - browser: { type: 'string' }, - }, - allowPositionals: true, - }) - - const testCommand = positionals[0] + const testCommand = process.argv[2] if (!testCommand) { - throw new Error('Usage: ci-bs.ts [--browser ]') + throw new Error('Usage: ci-bs.ts ') } const pr = await fetchPR(LOCAL_BRANCH!) @@ -42,16 +30,13 @@ runMain(async () => { return } - const environment: Record = { - BS_USERNAME: getBrowserStackUsername(), - BS_ACCESS_KEY: getBrowserStackAccessKey(), - } - - if (browser) { - environment.BS_BROWSER = browser - } - - command`yarn ${testCommand}:bs`.withEnvironment(environment).withLogs().run() + command`yarn ${testCommand}:bs` + .withEnvironment({ + BS_USERNAME: getBrowserStackUsername(), + BS_ACCESS_KEY: getBrowserStackAccessKey(), + }) + .withLogs() + .run() }) function hasRelevantChanges(baseCommit: string): boolean {