chore(ci): report transform benchmarks in CI (#918)

* perf(bundler): report transform benchmarks in CI

* fix(ci): validate transform benchmark results

Author: harlan-zw

.github/workflows/bundle-size.yml

@@ -32,17 +32,21 @@ jobs:
           cache: pnpm
 
       - name: Build base branch bundles
+        env:
+          BASE_REF: ${{ github.base_ref }}
         run: |
           # the PR's perf harness may not exist on the base ref; keep a copy so we can
           # measure the base build with the same script (same runner cancels variance)
           cp bench/perf-ci.mjs /tmp/perf-ci.mjs
           cp bench/bundle/dependency-analyze.mjs /tmp/dependency-analyze.mjs
+          cp bench/unplugin-transform.bench.ts /tmp/unplugin-transform.bench.ts
           # runner setup (pnpm store probing) can leave the lockfile dirty; nothing
           # in this fresh checkout is precious, so surface the diff and discard it
           git status --porcelain
           git diff --stat
-          git checkout -f ${{ github.base_ref }}
+          git checkout -f "$BASE_REF"
           pnpm install
+          cp /tmp/unplugin-transform.bench.ts bench/unplugin-transform.bench.ts
           node /tmp/dependency-analyze.mjs > /tmp/base-dependencies.json
           pnpm build
           # the base ref may predate some bundle configs (react, full, schema-org);
@@ -58,6 +62,7 @@ jobs:
           # measure base perf with the PR's harness against the base-built packages;
           # run from /tmp so the working tree stays clean across the checkout back
           node --expose-gc /tmp/perf-ci.mjs > /tmp/base-perf.json || echo '{}' > /tmp/base-perf.json
+          pnpm exec vitest bench bench/unplugin-transform.bench.ts --config bench/vitest.config.ts --run --outputJson /tmp/base-transform-perf.json
           # the base pnpm install may have rewritten the lockfile; discard before switching back
           git checkout -f -
 
@@ -71,6 +76,7 @@ jobs:
           pnpm test:react-bundle-size
           pnpm test:schema-org-bundle-size
           node --expose-gc bench/perf-ci.mjs > /tmp/pr-perf.json || echo '{}' > /tmp/pr-perf.json
+          pnpm exec vitest bench bench/unplugin-transform.bench.ts --config bench/vitest.config.ts --run --outputJson /tmp/pr-transform-perf.json
 
       - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
 
@@ -86,8 +92,10 @@ jobs:
           BASE_DIST: /tmp/base-dist
           BASE_DEPENDENCIES: /tmp/base-dependencies.json
           BASE_PERF: /tmp/base-perf.json
+          BASE_TRANSFORM_PERF: /tmp/base-transform-perf.json
           PR_DEPENDENCIES: /tmp/pr-dependencies.json
           PR_PERF: /tmp/pr-perf.json
+          PR_TRANSFORM_PERF: /tmp/pr-transform-perf.json
         run: |
           output=$(bun bench/bundle/report.ts)
           echo "$output"

bench/bundle/perf-report.test.ts

@@ -1,5 +1,47 @@
 import { describe, expect, it } from 'vitest'
-import { renderPerfReport } from './perf-report'
+import { parseVitestBenchmarks, renderPerfReport } from './perf-report'
+
+describe('parseVitestBenchmarks', () => {
+  it('treats a missing benchmark file as an empty run', () => {
+    expect(parseVitestBenchmarks(null)).toEqual({ benches: [] })
+  })
+
+  it('rejects malformed benchmark output', () => {
+    expect(() => parseVitestBenchmarks({
+      files: [{
+        groups: [{
+          benchmarks: [{
+            name: 'useSeoMetaTransform static calls',
+            mean: '1.25',
+            rme: 2.5,
+          }],
+        }],
+      }],
+    })).toThrowError('Invalid Vitest benchmark result')
+  })
+
+  it('converts transform benchmark output into performance benches', () => {
+    expect(parseVitestBenchmarks({
+      files: [{
+        groups: [{
+          benchmarks: [{
+            name: 'useSeoMetaTransform static calls',
+            mean: 1.25,
+            rme: 2.5,
+          }],
+        }],
+      }],
+    })).toEqual({
+      benches: [{
+        id: 'bundler-transform:useSeoMetaTransform static calls',
+        name: 'Bundler: useSeoMetaTransform static calls',
+        kind: 'time',
+        value: 1.25,
+        rme: 2.5,
+      }],
+    })
+  })
+})
 
 describe('renderPerfReport allocation noise gate', () => {
   it('keeps allocation changes within the combined RME out of the verdict', () => {

bench/bundle/perf-report.ts

@@ -23,6 +23,54 @@ export interface PerfRun {
   benches: PerfBench[]
 }
 
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null
+}
+
+function parseVitestBenchmark(value: unknown): PerfBench {
+  if (
+    !isRecord(value)
+    || typeof value.name !== 'string'
+    || typeof value.mean !== 'number'
+    || !Number.isFinite(value.mean)
+    || typeof value.rme !== 'number'
+    || !Number.isFinite(value.rme)
+  ) {
+    throw new TypeError('Invalid Vitest benchmark result')
+  }
+
+  return {
+    id: `bundler-transform:${value.name}`,
+    name: `Bundler: ${value.name}`,
+    kind: 'time',
+    value: value.mean,
+    rme: value.rme,
+  }
+}
+
+export function parseVitestBenchmarks(value: unknown): PerfRun {
+  if (value === null || value === undefined)
+    return { benches: [] }
+
+  if (!isRecord(value) || !Array.isArray(value.files))
+    throw new TypeError('Invalid Vitest benchmark output')
+
+  const benches = value.files.flatMap((file) => {
+    if (!isRecord(file) || !Array.isArray(file.groups))
+      throw new TypeError('Invalid Vitest benchmark file')
+    return file.groups.flatMap((group) => {
+      if (!isRecord(group) || !Array.isArray(group.benchmarks))
+        throw new TypeError('Invalid Vitest benchmark group')
+      return group.benchmarks.map(parseVitestBenchmark)
+    })
+  })
+
+  if (!benches.length)
+    throw new TypeError('Vitest benchmark output contained no results')
+
+  return { benches }
+}
+
 const TIME_FLOOR_PCT = 5
 const ALLOC_FLOOR_PCT = 2
 const ALLOC_FLOOR_BYTES = 1024

bench/bundle/report.ts

@@ -1,8 +1,9 @@
+import type { PerfRun } from './perf-report'
 import fs from 'node:fs'
 import process from 'node:process'
 import { collectBundleData, renderBundleReport } from './bundle-report'
 import { renderDependencyReport } from './dependency-report'
-import { renderPerfReport } from './perf-report'
+import { parseVitestBenchmarks, renderPerfReport } from './perf-report'
 
 // Combined bundle, dependency, and perf comment for the PR. Bundle data comes
 // from the dist dirs; dependency and perf data come from JSON generated for the
@@ -17,10 +18,22 @@ const prDependencies = readJson(process.env.PR_DEPENDENCIES)
 if (prDependencies?.packages?.length)
   sections.push(renderDependencyReport(readJson(process.env.BASE_DEPENDENCIES), prDependencies))
 
+function mergePerfRuns(...runs: Array<PerfRun | null>): PerfRun | null {
+  const benches = runs.flatMap(run => run?.benches || [])
+  return benches.length ? { benches } : null
+}
+
 // guard on benches: a perf run that failed writes `{}`, which must skip the section, not crash
-const prPerf = readJson(process.env.PR_PERF)
+const basePerf = mergePerfRuns(
+  readJson(process.env.BASE_PERF),
+  parseVitestBenchmarks(readJson(process.env.BASE_TRANSFORM_PERF)),
+)
+const prPerf = mergePerfRuns(
+  readJson(process.env.PR_PERF),
+  parseVitestBenchmarks(readJson(process.env.PR_TRANSFORM_PERF)),
+)
 if (prPerf?.benches?.length)
-  sections.push(renderPerfReport(readJson(process.env.BASE_PERF), prPerf))
+  sections.push(renderPerfReport(basePerf, prPerf))
 
 let out = sections.join('\n\n---\n\n')
 

bench/unplugin-transform.bench.ts

@@ -93,6 +93,21 @@ async function runPluginTransform(plugin: any, code: string, id: string, context
   return await transformHandler(plugin).call(context, code, id)
 }
 
+function assertTransformResult(result: unknown, name: string) {
+  if (
+    typeof result === 'string'
+    || (
+      typeof result === 'object'
+      && result !== null
+      && 'code' in result
+      && typeof (result as { code?: unknown }).code === 'string'
+    )
+  ) {
+    return
+  }
+  throw new TypeError(`${name} benchmark did not transform its fixture`)
+}
+
 describe('unplugin transform CPU', () => {
   bench('transformInclude mixed ids', () => {
     const seo = UseSeoMetaTransform.vite({}) as any
@@ -112,17 +127,25 @@ describe('unplugin transform CPU', () => {
 
   bench('useSeoMetaTransform static calls', async () => {
     const plugin = UseSeoMetaTransform.vite({}) as any
-    await runPluginTransform(plugin, seoCode, '/project/src/page.ts')
+    const result = await runPluginTransform(plugin, seoCode, '/project/src/page.ts')
+    assertTransformResult(result, plugin.name)
   })
 
   bench('minifyTransform inline script/style', async () => {
     const plugin = MinifyTransform.vite({ js: mockJSMinifier, css: mockCSSMinifier }) as any
-    await runPluginTransform(plugin, minifyCode, '/project/src/page.ts')
+    const result = await runPluginTransform(plugin, minifyCode, '/project/src/page.ts')
+    assertTransformResult(result, plugin.name)
   })
 
   bench('treeshakeServerComposables many calls', async () => {
     const plugin = TreeshakeServerComposables.vite({}) as any
-    await runPluginTransform(plugin, treeshakeCode, '/project/src/page.ts')
+    const result = await runPluginTransform(
+      plugin,
+      treeshakeCode,
+      '/project/src/page.ts',
+      { environment: { config: { consumer: 'client' } } },
+    )
+    assertTransformResult(result, plugin.name)
   })
 
   bench('treeshakeServerComposables skip unrelated code', async () => {
@@ -133,7 +156,8 @@ describe('unplugin transform CPU', () => {
   bench('ssrStaticReplace many head.ssr reads', async () => {
     const plugin = SSRStaticReplace.vite({}) as any
     plugin.apply({}, { command: 'build', isSsrBuild: false })
-    await runPluginTransform(plugin, ssrStaticReplaceCode, '/project/node_modules/unhead/dist/index.mjs')
+    const result = await runPluginTransform(plugin, ssrStaticReplaceCode, '/project/node_modules/unhead/dist/index.mjs')
+    assertTransformResult(result, plugin.name)
   })
 
   bench('ssrStaticReplace skip unrelated code', async () => {
@@ -154,7 +178,8 @@ describe('unplugin transform CPU', () => {
     })
     const plugin = CreateHeadTransform(ctx) as any
     plugin.configResolved({ root: '/project' })
-    await plugin.transform.handler.call({ environment: { config: { consumer: 'client' } } }, createHeadCode, '/project/src/head.ts')
+    const result = await plugin.transform.handler.call({ environment: { config: { consumer: 'client' } } }, createHeadCode, '/project/src/head.ts')
+    assertTransformResult(result, plugin.name)
   })
 
   bench('react streaming skip JSX without head calls', async () => {
@@ -164,7 +189,8 @@ describe('unplugin transform CPU', () => {
 
   bench('react streaming transform JSX with head calls', async () => {
     const plugin = unheadReactStreamingPlugin.vite({}) as any
-    await plugin.transform.handler.call({ environment: { name: 'client' } }, jsxWithHeadCode, '/project/src/page.tsx')
+    const result = await plugin.transform.handler.call({ environment: { name: 'client' } }, jsxWithHeadCode, '/project/src/page.tsx')
+    assertTransformResult(result, plugin.name)
   })
 
   bench('solid streaming skip JSX without head calls', async () => {