From 5b4a37d943593545cf3d9d5ffd085c9ecae98e91 Mon Sep 17 00:00:00 2001 From: Rom Grk Date: Thu, 11 Jun 2026 09:19:52 -0400 Subject: [PATCH 1/2] feat: add interactive DevTools profiling mode to benchmark package Add a profile mode to `@mui/internal-benchmark` that replaces the automated measurement loop with an interactive, headed browser session for hand-driving the DevTools profiler. Enabled via `createBenchmarkVitestConfig({ profile: true })` or `BENCHMARK_PROFILE=true`. Each `benchmark()` case renders a control panel with a Render/Unmount toggle, an optional Run interaction button, and Finish. Profile runs drop the determinism V8/GPU flags, render into a full desktop viewport, auto-open DevTools, and use a 1-year test timeout since they are driven by hand. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/benchmark/README.md | 34 ++++ packages/benchmark/src/index.tsx | 293 +++++++++++++++++++++++++------ packages/benchmark/src/vitest.ts | 119 ++++++++++--- 3 files changed, 370 insertions(+), 76 deletions(-) diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index f4c980bcd..d862e4cf9 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -110,6 +110,38 @@ benchmark('name', renderFn, interaction, { vitest run ``` +### Profiling in DevTools + +To profile a benchmark case by hand with the browser DevTools instead of running the automated measurement loop, enable profile mode. It opens a **headed** Chromium window with DevTools already open, and replaces the measurement loop with an interactive control panel: + +```bash +BENCHMARK_PROFILE=true vitest run -t "MyComponent mount" +``` + +Each `benchmark()` case renders a toolbar pinned to the top of the page with **Render**, **Finish**, and (when the case has an interaction) **Run interaction** buttons. The **Render** button toggles between mounting and unmounting (it reads **Unmount** while the component is mounted). The component under test stays unmounted until you click **Render**, so the flow is: + +1. Switch to the DevTools **Performance** tab and start recording. +2. Click **Render** — this mounts the component (the thing you're profiling). +3. Stop the recording and inspect. Toggle **Unmount** / **Render** to capture more frames, or **Run interaction** to profile a re-render. +4. Click **Finish** to end the case and move to the next one. + +Filter to a single case with Vitest's `-t ""` (or by file) so the window isn't shared across many cases. Profile runs drop the deterministic V8 flags and software rendering used for measurement (`--no-opt`, `--predictable`, `--disable-gpu`, …) so the numbers in the profiler reflect realistic performance; they are therefore not comparable to measurement-mode results. + +Profile mode renders into a full desktop viewport (1920x1080 by default) and sizes the browser window to match, instead of Vitest's phone-sized 414x896 default. Set it to your screen resolution to fill the whole window, via the `profileViewport` option or the `BENCHMARK_PROFILE_VIEWPORT` env var: + +```bash +BENCHMARK_PROFILE=true BENCHMARK_PROFILE_VIEWPORT=2560x1440 vitest run -t "MyComponent mount" +``` + +Profile mode is also settable via the `profile` config option: + +```ts +export default createBenchmarkVitestConfig({ + profile: true, + profileViewport: { width: 2560, height: 1440 }, +}); +``` + ### Configuration `createBenchmarkVitestConfig` accepts: @@ -117,6 +149,8 @@ vitest run - `outputPath` — path for JSON results (default: `benchmarks/results.json`). Also settable via `BENCHMARK_OUTPUT_PATH`. - `baselinePath` — path to a prior results JSON file to inline as the comparison base (see [Baseline comparisons](#baseline-comparisons)). Also settable via `BENCHMARK_BASELINE_PATH`. - `launchArgs` — additional browser launch arguments +- `profile` — run an interactive profiling session in a headed browser with DevTools instead of measuring (see [Profiling in DevTools](#profiling-in-devtools)). Also settable via `BENCHMARK_PROFILE=true`. +- `profileViewport` — `{ width, height }` viewport (and window size) for profile mode (default: `1920x1080`). Also settable via `BENCHMARK_PROFILE_VIEWPORT` (e.g. `2560x1440`). To override standard Vitest options (e.g. `include`, `testTimeout`, `headless`), use `mergeConfig`: diff --git a/packages/benchmark/src/index.tsx b/packages/benchmark/src/index.tsx index b9e77223e..8e4d85ebe 100644 --- a/packages/benchmark/src/index.tsx +++ b/packages/benchmark/src/index.tsx @@ -65,12 +65,237 @@ function supportsElementTiming(): boolean { return PerformanceObserver.supportedEntryTypes.includes('element'); } +// When true, `benchmark()` opens an interactive profiling session in a headed +// browser instead of running the automated measurement loop. Enabled by +// `createBenchmarkVitestConfig({ profile: true })` or `BENCHMARK_PROFILE=true`, +// both of which replace this expression at build time via Vite `define`. +const PROFILE_MODE = process.env.BENCHMARK_PROFILE === 'true'; + +interface ElementTimingWaiter { + elementEntries: PerformanceElementTiming[]; + waitForElementTiming: (identifier: string, timeout?: number) => Promise; + disconnect: () => void; +} + +// Sets up a PerformanceObserver for the Element Timing API and exposes a +// promise-based `waitForElementTiming` helper. Shared by the measurement loop +// and the interactive profiling session. +function createElementTimingWaiter(): ElementTimingWaiter { + const hasElementTiming = supportsElementTiming(); + const elementEntries: PerformanceElementTiming[] = []; + const elementResolvers = new Map void>(); + + let observer: PerformanceObserver | null = null; + if (hasElementTiming) { + observer = new PerformanceObserver((list) => { + for (const entry of list.getEntries() as PerformanceElementTiming[]) { + elementEntries.push(entry); + const resolver = elementResolvers.get(entry.identifier); + if (resolver) { + elementResolvers.delete(entry.identifier); + resolver(); + } + } + }); + observer.observe({ type: 'element', buffered: false }); + } + + const waitForElementTiming = (identifier: string, timeout?: number): Promise => { + if (!hasElementTiming) { + console.warn( + `waitForElementTiming("${identifier}"): Element Timing API is not supported. ` + + 'Paint metrics will not be collected.', + ); + return Promise.resolve(); + } + if (elementEntries.some((entry) => entry.identifier === identifier)) { + return Promise.resolve(); + } + const { promise, resolve, reject } = Promise.withResolvers(); + const timeoutMs = timeout ?? 5000; + const timer = + timeoutMs > 0 && timeoutMs < Infinity + ? setTimeout(() => { + elementResolvers.delete(identifier); + reject( + new Error( + `waitForElementTiming("${identifier}"): timed out after ${timeoutMs}ms. ` + + 'Ensure the element has an `elementtiming` attribute and is visible in the viewport.', + ), + ); + }, timeoutMs) + : undefined; + elementResolvers.set(identifier, () => { + if (timer) { + clearTimeout(timer); + } + resolve(); + }); + return promise; + }; + + return { + elementEntries, + waitForElementTiming, + disconnect: () => observer?.disconnect(), + }; +} + interface BenchmarkOptions { runs?: number; warmupRuns?: number; afterEach?: () => Promise | void; } +const PROFILE_PANEL_STYLE = [ + 'position:fixed', + 'top:0', + 'left:0', + 'right:0', + 'z-index:2147483647', + 'display:flex', + 'gap:8px', + 'align-items:center', + 'box-sizing:border-box', + 'padding:8px 12px', + 'background:#1e1e1e', + 'color:#fff', + 'font:13px/1.4 ui-monospace,SFMono-Regular,Menlo,Consolas,monospace', + 'box-shadow:0 1px 6px rgba(0,0,0,0.5)', +].join(';'); + +const PROFILE_BUTTON_STYLE = [ + 'padding:4px 10px', + 'border:1px solid #555', + 'border-radius:4px', + 'background:#333', + 'color:#fff', + 'cursor:pointer', + 'font:inherit', +].join(';'); + +function createProfileButton(label: string): HTMLButtonElement { + const button = document.createElement('button'); + button.type = 'button'; + button.textContent = label; + button.style.cssText = PROFILE_BUTTON_STYLE; + return button; +} + +// Interactive profiling session: instead of measuring, render a control panel +// with Render / Unmount / Run interaction / Finish buttons. The component under +// test stays unmounted until the user clicks "Render", giving them time to start +// the DevTools profiler first. The returned promise resolves on "Finish", which +// is what keeps the Vitest test (and the headed browser window) alive in between. +function runProfileSession( + name: string, + renderFn: () => React.ReactElement, + interaction?: (ctx: InteractionContext) => Promise | void, +): Promise { + const panel = document.createElement('div'); + panel.setAttribute('data-benchmark-profile-panel', ''); + panel.style.cssText = PROFILE_PANEL_STYLE; + + const title = document.createElement('span'); + title.textContent = `⏱ ${name}`; + title.style.cssText = 'font-weight:600;white-space:nowrap'; + + const status = document.createElement('span'); + status.style.cssText = 'margin-left:auto;opacity:0.85;white-space:nowrap'; + + const renderButton = createProfileButton('▶ Render'); + const interactButton = interaction ? createProfileButton('⚡ Run interaction') : null; + const finishButton = createProfileButton('✓ Finish'); + + if (interactButton) { + interactButton.disabled = true; + } + + panel.appendChild(title); + panel.appendChild(renderButton); + if (interactButton) { + panel.appendChild(interactButton); + } + panel.appendChild(finishButton); + panel.appendChild(status); + document.body.appendChild(panel); + + // Push page content below the fixed panel so it doesn't cover the component. + const spacer = document.createElement('div'); + spacer.style.height = `${panel.offsetHeight}px`; + document.body.insertBefore(spacer, panel); + + const container = document.createElement('div'); + document.body.appendChild(container); + + const timing = createElementTimingWaiter(); + let root: ReactDOMClient.Root | null = null; + + const setStatus = (text: string) => { + status.textContent = text; + }; + + const show = () => { + if (root) { + return; + } + root = ReactDOMClient.createRoot(container); + ReactDOM.flushSync(() => { + root!.render(renderFn()); + }); + renderButton.textContent = '■ Unmount'; + if (interactButton) { + interactButton.disabled = false; + } + setStatus('rendered — capture your profile, then Unmount or Finish'); + }; + + const hide = () => { + if (!root) { + return; + } + root.unmount(); + root = null; + renderButton.textContent = '▶ Render'; + if (interactButton) { + interactButton.disabled = true; + } + setStatus('unmounted — Render again or Finish'); + }; + + setStatus('idle — start the DevTools profiler, then click Render'); + + return new Promise((resolve) => { + renderButton.addEventListener('click', () => (root ? hide() : show())); + + if (interactButton && interaction) { + interactButton.addEventListener('click', async () => { + interactButton.disabled = true; + setStatus('running interaction…'); + try { + await interaction({ waitForElementTiming: timing.waitForElementTiming }); + setStatus('interaction done'); + } catch (error) { + setStatus(`interaction error: ${String(error)}`); + } finally { + if (root) { + interactButton.disabled = false; + } + } + }); + } + + finishButton.addEventListener('click', () => { + hide(); + timing.disconnect(); + spacer.remove(); + container.remove(); + panel.remove(); + resolve(); + }); + }); +} + export function benchmark( name: string, renderFn: () => React.ReactElement, @@ -80,6 +305,15 @@ export function benchmark( const interaction = typeof interactionOrOptions === 'function' ? interactionOrOptions : undefined; const options = typeof interactionOrOptions === 'object' ? interactionOrOptions : maybeOptions; + // In profile mode, skip the automated measurement loop entirely and hand the + // case to an interactive session so a human can drive the DevTools profiler. + if (PROFILE_MODE) { + it(name, async () => { + await runProfileSession(name, renderFn, interaction); + }); + return; + } + it(name, async ({ task }) => { const runs = options?.runs ?? 20; const warmupRuns = options?.warmupRuns ?? 10; @@ -87,8 +321,6 @@ export function benchmark( const totalRuns = warmupRuns + runs; const iterations: IterationData[] = []; - const hasElementTiming = supportsElementTiming(); - if (typeof window.gc !== 'function') { console.warn( 'window.gc is not available. Run with --js-flags=--expose-gc for consistent GC between iterations.', @@ -106,58 +338,7 @@ export function benchmark( forceGC(); const captures: RenderEvent[] = []; - const elementEntries: PerformanceElementTiming[] = []; - const elementResolvers = new Map void>(); - - // Set up Element Timing observer - let elementObserver: PerformanceObserver | null = null; - if (hasElementTiming) { - elementObserver = new PerformanceObserver((list) => { - for (const entry of list.getEntries() as PerformanceElementTiming[]) { - elementEntries.push(entry); - const resolver = elementResolvers.get(entry.identifier); - if (resolver) { - elementResolvers.delete(entry.identifier); - resolver(); - } - } - }); - elementObserver.observe({ type: 'element', buffered: false }); - } - - const waitForElementTiming = (identifier: string, timeout?: number): Promise => { - if (!hasElementTiming) { - console.warn( - `waitForElementTiming("${identifier}"): Element Timing API is not supported. ` + - 'Paint metrics will not be collected.', - ); - return Promise.resolve(); - } - if (elementEntries.some((entry) => entry.identifier === identifier)) { - return Promise.resolve(); - } - const { promise, resolve, reject } = Promise.withResolvers(); - const timeoutMs = timeout ?? 5000; - const timer = - timeoutMs > 0 && timeoutMs < Infinity - ? setTimeout(() => { - elementResolvers.delete(identifier); - reject( - new Error( - `waitForElementTiming("${identifier}"): timed out after ${timeoutMs}ms. ` + - 'Ensure the element has an `elementtiming` attribute and is visible in the viewport.', - ), - ); - }, timeoutMs) - : undefined; - elementResolvers.set(identifier, () => { - if (timer) { - clearTimeout(timer); - } - resolve(); - }); - return promise; - }; + const { elementEntries, waitForElementTiming, disconnect } = createElementTimingWaiter(); const iterationStart = performance.now(); @@ -176,7 +357,7 @@ export function benchmark( }); if (renderError) { - elementObserver?.disconnect(); + disconnect(); root.unmount(); container.remove(); break; @@ -191,7 +372,7 @@ export function benchmark( // eslint-disable-next-line no-await-in-loop await waitForElementTiming('default', 0); - elementObserver?.disconnect(); + disconnect(); root.unmount(); container.remove(); diff --git a/packages/benchmark/src/vitest.ts b/packages/benchmark/src/vitest.ts index 3255a93c1..bd9100d4b 100644 --- a/packages/benchmark/src/vitest.ts +++ b/packages/benchmark/src/vitest.ts @@ -18,17 +18,95 @@ export interface CreateBenchmarkVitestConfigOptions { * Additional Chromium launch arguments. */ launchArgs?: string[]; + /** + * Run each `benchmark()` case as an interactive profiling session in a headed + * browser instead of the automated measurement loop. Each case renders a + * control panel with Render / Unmount / Finish buttons so you can start the + * DevTools profiler before the component mounts. The deterministic V8 flags + * and software rendering used for measurement are dropped so the numbers in + * the profiler reflect realistic performance. Also settable via + * `BENCHMARK_PROFILE=true`. + */ + profile?: boolean; + /** + * Viewport (and matching browser window size) used in profile mode. Vitest's + * default browser viewport is a phone-sized 414x896; profiling overrides it + * with a full desktop viewport so the component renders at a realistic size. + * Set this to your screen's resolution to fill the whole window. Also settable + * via `BENCHMARK_PROFILE_VIEWPORT` (e.g. `2560x1440`). Defaults to 1920x1080. + */ + profileViewport?: { width: number; height: number }; } +// Resolves the profile-mode viewport from the option, the +// `BENCHMARK_PROFILE_VIEWPORT` env var (`x`), or a 1920x1080 +// default. +function resolveProfileViewport(option?: { + width: number; + height: number; +}): { width: number; height: number } { + if (option) { + return option; + } + const env = process.env.BENCHMARK_PROFILE_VIEWPORT; + const match = env ? /^\s*(\d+)\s*[x×]\s*(\d+)\s*$/.exec(env) : null; + if (match) { + return { width: Number(match[1]), height: Number(match[2]) }; + } + return { width: 1920, height: 1080 }; +} + +// Determinism flags used for measurement runs. They make timings reproducible +// but are NOT representative of real performance (`--no-opt` disables the +// optimizing compiler, `--disable-gpu` forces software rendering), so profiling +// runs use a lighter set instead. +const MEASUREMENT_LAUNCH_ARGS = [ + // V8 flags for deterministic JS execution + '--js-flags=--expose-gc,--predictable,--no-opt,--predictable-gc-schedule,--no-concurrent-sweeping,--hash-seed=1,--random-seed=1,--max-old-space-size=4096', + + // Chromium flags to reduce renderer/compositor noise + '--disable-background-timer-throttling', + '--disable-backgrounding-occluded-windows', + '--disable-renderer-backgrounding', + '--disable-background-networking', + // Reduces environmental noise by disabling field trials, + // for more consistent profiling results. + '--enable-benchmarking', + // Forces software rendering instead of GPU, which is more deterministic. + '--disable-gpu', +]; + +// Launch args for interactive profiling: keep GC exposed and reduce background +// throttling noise, but let V8 optimize and the GPU render normally, and open +// DevTools automatically so the profiler is one click away. +const PROFILE_LAUNCH_ARGS = [ + '--js-flags=--expose-gc', + '--disable-background-timer-throttling', + '--disable-backgrounding-occluded-windows', + '--disable-renderer-backgrounding', + '--auto-open-devtools-for-tabs', +]; + export function createBenchmarkVitestConfig( options?: CreateBenchmarkVitestConfigOptions, ): ViteUserConfig { const { outputPath, baselinePath, launchArgs = [] } = options ?? {}; + const profile = options?.profile ?? process.env.BENCHMARK_PROFILE === 'true'; + const profileViewport = resolveProfileViewport(options?.profileViewport); + + // Size the actual browser window to the viewport too: Vitest's `viewport` only + // sizes the test iframe, so without this the headed window stays small and the + // full-size iframe is cropped/scrolled instead of filling the window. + const profileArgs = [ + ...PROFILE_LAUNCH_ARGS, + `--window-size=${profileViewport.width},${profileViewport.height}`, + ]; return { plugins: [react()], define: { 'process.env.NODE_ENV': '"production"', + 'process.env.BENCHMARK_PROFILE': JSON.stringify(profile ? 'true' : ''), }, resolve: { dedupe: ['react', 'react-dom'], @@ -37,33 +115,34 @@ export function createBenchmarkVitestConfig( test: { browser: { enabled: true, - headless: true, + headless: !profile, + // Profiling renders into a clean page: hide Vitest's browser runner UI + // so the orchestrator chrome doesn't clutter what you're profiling. + ui: profile ? false : undefined, + // Vitest's default browser viewport is a phone-sized 414x896. For + // profiling, render into a full desktop viewport instead. (Measurement + // keeps the default so results stay comparable.) + viewport: profile ? profileViewport : undefined, screenshotFailures: false, - instances: [{ browser: 'chromium', testTimeout: 120_000 }], + instances: [ + { + browser: 'chromium', + // Profiling sessions are driven by hand, so give them effectively + // unlimited time instead of the measurement timeout. + testTimeout: profile ? 365 * 24 * 60 * 60 * 1000 : 120_000, + }, + ], provider: playwright({ launchOptions: { - args: [ - // V8 flags for deterministic JS execution - '--js-flags=--expose-gc,--predictable,--no-opt,--predictable-gc-schedule,--no-concurrent-sweeping,--hash-seed=1,--random-seed=1,--max-old-space-size=4096', - - // Chromium flags to reduce renderer/compositor noise - '--disable-background-timer-throttling', - '--disable-backgrounding-occluded-windows', - '--disable-renderer-backgrounding', - '--disable-background-networking', - // Reduces environmental noise by disabling field trials, - // for more consistent profiling results. - '--enable-benchmarking', - // Forces software rendering instead of GPU, which is more deterministic. - '--disable-gpu', - - ...launchArgs, - ], + args: [...(profile ? profileArgs : MEASUREMENT_LAUNCH_ARGS), ...launchArgs], }, }), }, fileParallelism: false, - reporters: ['default', ['@mui/internal-benchmark/reporter', { outputPath, baselinePath }]], + // Profiling sessions don't measure anything, so skip the results reporter. + reporters: profile + ? ['default'] + : ['default', ['@mui/internal-benchmark/reporter', { outputPath, baselinePath }]], include: ['**/*.bench.tsx'], }, }; From a577a57e9e9e70ca419f76fa195f8fabf1e09e7e Mon Sep 17 00:00:00 2001 From: Rom Grk Date: Thu, 11 Jun 2026 10:13:06 -0400 Subject: [PATCH 2/2] lint --- packages/benchmark/src/vitest.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/benchmark/src/vitest.ts b/packages/benchmark/src/vitest.ts index bd9100d4b..14aec4b0c 100644 --- a/packages/benchmark/src/vitest.ts +++ b/packages/benchmark/src/vitest.ts @@ -41,10 +41,10 @@ export interface CreateBenchmarkVitestConfigOptions { // Resolves the profile-mode viewport from the option, the // `BENCHMARK_PROFILE_VIEWPORT` env var (`x`), or a 1920x1080 // default. -function resolveProfileViewport(option?: { +function resolveProfileViewport(option?: { width: number; height: number }): { width: number; height: number; -}): { width: number; height: number } { +} { if (option) { return option; }