diff --git a/.github/scripts/firefox-buildwebdriver-benchmark.ts b/.github/scripts/firefox-buildwebdriver-benchmark.ts new file mode 100644 index 000000000000..284e83d8aae3 --- /dev/null +++ b/.github/scripts/firefox-buildwebdriver-benchmark.ts @@ -0,0 +1,318 @@ +import fs from 'node:fs'; +import { tmpdir } from 'node:os'; +import path, { join } from 'node:path'; +import { performance } from 'node:perf_hooks'; +import { pathToFileURL } from 'node:url'; +import { createRequire } from 'node:module'; + +type Args = { + iterations: number; + out?: string; + refLabel: string; +}; + +type PhaseSample = { + addonSetupMs: number; + builderBuildMs: number; + getInternalIdMs: number; + installExtensionMs: number; + totalMs: number; +}; + +type ScenarioName = 'cold' | 'warmUnchanged' | 'warmManifestUpdate'; + +type PhaseSummary = { + mean: number; + median: number; + p95: number; +}; + +type ScenarioResult = { + samples: PhaseSample[]; + summary: Record; +}; + +type BenchmarkResult = { + gitRef: string; + gitSha: string; + iterations: number; + scenarios: Record; +}; + +function parseArgs(argv: string[]): Args { + let iterations = 3; + let out: string | undefined; + let refLabel = 'unknown'; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === '--iterations') { + iterations = Number(argv[index + 1]); + index += 1; + } else if (arg === '--out') { + out = argv[index + 1]; + index += 1; + } else if (arg === '--ref-label') { + refLabel = argv[index + 1]; + index += 1; + } else { + throw new Error(`Unknown argument: ${arg}`); + } + } + + if (!Number.isInteger(iterations) || iterations < 1) { + throw new Error(`Invalid iterations: ${iterations}`); + } + + return { iterations, out, refLabel }; +} + +function removeXpiCaches() { + for (const entry of fs.readdirSync(tmpdir())) { + if (entry.startsWith('metamask-e2e-')) { + fs.rmSync(join(tmpdir(), entry), { force: true, recursive: true }); + } + } +} + +function mean(values: number[]) { + return values.reduce((sum, value) => sum + value, 0) / values.length; +} + +function percentile(values: number[], percentileValue: number) { + const sorted = [...values].sort((left, right) => left - right); + const index = Math.min( + sorted.length - 1, + Math.ceil((percentileValue / 100) * sorted.length) - 1, + ); + return sorted[index]; +} + +function summarizePhase(values: number[]): PhaseSummary { + return { + mean: mean(values), + median: percentile(values, 50), + p95: percentile(values, 95), + }; +} + +function summarize(samples: PhaseSample[]) { + return { + addonSetupMs: summarizePhase( + samples.map(({ addonSetupMs }) => addonSetupMs), + ), + builderBuildMs: summarizePhase( + samples.map(({ builderBuildMs }) => builderBuildMs), + ), + getInternalIdMs: summarizePhase( + samples.map(({ getInternalIdMs }) => getInternalIdMs), + ), + installExtensionMs: summarizePhase( + samples.map(({ installExtensionMs }) => installExtensionMs), + ), + totalMs: summarizePhase(samples.map(({ totalMs }) => totalMs)), + }; +} + +function formatMs(value: number) { + return `${value.toFixed(1)} ms`; +} + +async function main() { + const { iterations, out, refLabel } = parseArgs(process.argv.slice(2)); + const root = process.cwd(); + const requireFromTarget = createRequire(path.join(root, 'package.json')); + + process.env.SELENIUM_BROWSER = 'firefox'; + process.env.HEADLESS = 'false'; + + const { Builder } = requireFromTarget( + 'selenium-webdriver', + ) as typeof import('selenium-webdriver'); + const FirefoxDriver = requireFromTarget('./test/e2e/webdriver/firefox') as { + prototype: { + getInternalId(...args: unknown[]): Promise; + installExtension(...args: unknown[]): Promise; + }; + }; + const { buildWebDriver } = requireFromTarget('./test/e2e/webdriver') as { + buildWebDriver(options?: Record): Promise<{ + driver: { quit(): Promise }; + }>; + }; + const { setManifestFlags } = (await import( + pathToFileURL(join(root, 'test/e2e/set-manifest-flags.ts')).href + )) as { + setManifestFlags(flags?: Record): Promise; + }; + + let currentSample: Omit | null = + null; + + const originalBuilderBuild = Builder.prototype.build; + const originalInstallExtension = FirefoxDriver.prototype.installExtension; + const originalGetInternalId = FirefoxDriver.prototype.getInternalId; + + Builder.prototype.build = function patchedBuild(...args: unknown[]) { + const start = performance.now(); + try { + return originalBuilderBuild.apply(this, args); + } finally { + if (currentSample) { + currentSample.builderBuildMs += performance.now() - start; + } + } + }; + + FirefoxDriver.prototype.installExtension = + async function patchedInstallExtension(...args: unknown[]) { + const start = performance.now(); + try { + return await originalInstallExtension.apply(this, args); + } finally { + if (currentSample) { + currentSample.installExtensionMs += performance.now() - start; + } + } + }; + + FirefoxDriver.prototype.getInternalId = async function patchedGetInternalId( + ...args: unknown[] + ) { + const start = performance.now(); + try { + return await originalGetInternalId.apply(this, args); + } finally { + if (currentSample) { + currentSample.getInternalIdMs += performance.now() - start; + } + } + }; + + async function measureBuildWebDriver(): Promise { + currentSample = { + builderBuildMs: 0, + getInternalIdMs: 0, + installExtensionMs: 0, + }; + const start = performance.now(); + const { driver } = await buildWebDriver(); + const totalMs = performance.now() - start; + await driver.quit(); + const sample = { + ...currentSample, + addonSetupMs: Math.max( + 0, + totalMs - + currentSample.builderBuildMs - + currentSample.installExtensionMs - + currentSample.getInternalIdMs, + ), + totalMs, + }; + currentSample = null; + return sample; + } + + async function primeAndMeasure( + initialFlags: Record, + measuredFlags: Record, + ) { + removeXpiCaches(); + await setManifestFlags(initialFlags); + await measureBuildWebDriver(); + await setManifestFlags(measuredFlags); + return await measureBuildWebDriver(); + } + + const scenarios: Record = { + cold: [], + warmManifestUpdate: [], + warmUnchanged: [], + }; + + for (let iteration = 0; iteration < iterations; iteration += 1) { + const seed = `${refLabel}-${iteration}`; + + removeXpiCaches(); + await setManifestFlags({ xpiBenchmarkVariant: `cold-${seed}` }); + scenarios.cold.push(await measureBuildWebDriver()); + + scenarios.warmUnchanged.push( + await primeAndMeasure( + { xpiBenchmarkVariant: `warm-hit-${seed}` }, + { xpiBenchmarkVariant: `warm-hit-${seed}` }, + ), + ); + + scenarios.warmManifestUpdate.push( + await primeAndMeasure( + { xpiBenchmarkVariant: `warm-before-${seed}` }, + { xpiBenchmarkVariant: `warm-after-${seed}` }, + ), + ); + } + + const gitSha = requireFromTarget('node:child_process') + .execFileSync('git', ['rev-parse', 'HEAD'], { cwd: root }) + .toString() + .trim(); + + const result: BenchmarkResult = { + gitRef: refLabel, + gitSha, + iterations, + scenarios: { + cold: { + samples: scenarios.cold, + summary: summarize(scenarios.cold), + }, + warmManifestUpdate: { + samples: scenarios.warmManifestUpdate, + summary: summarize(scenarios.warmManifestUpdate), + }, + warmUnchanged: { + samples: scenarios.warmUnchanged, + summary: summarize(scenarios.warmUnchanged), + }, + }, + }; + + console.log(`Firefox buildWebDriver benchmark for ${refLabel} (${gitSha})`); + console.log( + 'Scenario'.padEnd(22) + + 'metric'.padStart(10) + + 'builder.build'.padStart(16) + + 'addonSetup'.padStart(16) + + 'installExtension'.padStart(20) + + 'getInternalId'.padStart(16) + + 'total'.padStart(12), + ); + + for (const [name, scenario] of Object.entries(result.scenarios) as [ + ScenarioName, + ScenarioResult, + ][]) { + for (const metric of ['mean', 'median', 'p95'] as const) { + console.log( + (metric === 'mean' ? name : '').padEnd(22) + + metric.padStart(10) + + formatMs(scenario.summary.builderBuildMs[metric]).padStart(16) + + formatMs(scenario.summary.addonSetupMs[metric]).padStart(16) + + formatMs(scenario.summary.installExtensionMs[metric]).padStart(20) + + formatMs(scenario.summary.getInternalIdMs[metric]).padStart(16) + + formatMs(scenario.summary.totalMs[metric]).padStart(12), + ); + } + } + + if (out) { + fs.mkdirSync(path.dirname(out), { recursive: true }); + fs.writeFileSync(out, JSON.stringify(result, null, 2)); + } +} + +main().catch((error: unknown) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/.github/scripts/firefox-installaddon-variant-benchmark.ts b/.github/scripts/firefox-installaddon-variant-benchmark.ts new file mode 100644 index 000000000000..a1e54c5fac61 --- /dev/null +++ b/.github/scripts/firefox-installaddon-variant-benchmark.ts @@ -0,0 +1,271 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { performance } from 'node:perf_hooks'; +import { createHash } from 'node:crypto'; +import { execFileSync } from 'node:child_process'; +import { createRequire } from 'node:module'; + +type Args = { + iterations: number; + out?: string; + refLabel: string; +}; + +type VariantName = + | 'zip1' + | 'yazlCurrent' + | 'yazlManifestCompressed' + | 'yazlManifestStoredRaw'; + +type VariantResult = { + path: string; + size: number; + samples: number[]; + summary: { + mean: number; + median: number; + p95: number; + }; +}; + +type BenchmarkResult = { + gitRef: string; + gitSha: string; + iterations: number; + variants: Record; +}; + +const MANIFEST_FILE_NAME = 'manifest.json'; +const MANIFEST_SIZE = 64 * 1024; + +function parseArgs(argv: string[]): Args { + let iterations = 7; + let out: string | undefined; + let refLabel = 'unknown'; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === '--iterations') { + iterations = Number(argv[index + 1]); + index += 1; + } else if (arg === '--out') { + out = argv[index + 1]; + index += 1; + } else if (arg === '--ref-label') { + refLabel = argv[index + 1]; + index += 1; + } else { + throw new Error(`Unknown argument: ${arg}`); + } + } + + if (!Number.isInteger(iterations) || iterations < 1) { + throw new Error(`Invalid iterations: ${iterations}`); + } + + return { iterations, out, refLabel }; +} + +function mean(values: number[]) { + return values.reduce((sum, value) => sum + value, 0) / values.length; +} + +function percentile(values: number[], percentileValue: number) { + const sorted = [...values].sort((left, right) => left - right); + const index = Math.min( + sorted.length - 1, + Math.ceil((percentileValue / 100) * sorted.length) - 1, + ); + return sorted[index]; +} + +function summarize(values: number[]) { + return { + mean: mean(values), + median: percentile(values, 50), + p95: percentile(values, 95), + }; +} + +function formatMs(value: number) { + return `${value.toFixed(1)} ms`; +} + +function buildPaddedManifest(absExtDir: string) { + const manifest = Buffer.allocUnsafe(MANIFEST_SIZE); + manifest.fill( + 0x20, + fs.readFileSync(path.join(absExtDir, MANIFEST_FILE_NAME)).copy(manifest), + ); + return manifest; +} + +async function main() { + const { iterations, out, refLabel } = parseArgs(process.argv.slice(2)); + const root = process.cwd(); + const requireFromTarget = createRequire(path.join(root, 'package.json')); + const { Builder } = requireFromTarget( + 'selenium-webdriver', + ) as typeof import('selenium-webdriver'); + const firefox = requireFromTarget( + 'selenium-webdriver/firefox', + ) as typeof import('selenium-webdriver/firefox'); + const { ZipFile } = requireFromTarget('yazl') as typeof import('yazl'); + + const absExtDir = path.resolve('dist/firefox'); + const rawManifest = fs.readFileSync(path.join(absExtDir, MANIFEST_FILE_NAME)); + const paddedManifest = buildPaddedManifest(absExtDir); + const benchmarkDir = fs.mkdtempSync( + path.join(os.tmpdir(), 'firefox-installaddon-variants-'), + ); + + function buildZip1Xpi() { + const xpiPath = path.join(benchmarkDir, 'zip1.xpi'); + execFileSync('zip', ['-r', '-1', '-q', xpiPath, '.'], { cwd: absExtDir }); + return xpiPath; + } + + async function buildYazlXpi( + name: string, + manifestBuffer: Buffer, + manifestOptions: { + compress: boolean; + compressionLevel?: number; + }, + ) { + const xpiPath = path.join(benchmarkDir, `${name}.xpi`); + const zipFile = new ZipFile(); + const manifestHash = createHash('sha256') + .update(manifestBuffer) + .digest('base64'); + + zipFile.addBuffer(manifestBuffer, MANIFEST_FILE_NAME, manifestOptions); + for (const entry of fs.readdirSync(absExtDir, { + recursive: true, + withFileTypes: true, + })) { + if (entry.isFile()) { + const absPath = path.join(entry.parentPath, entry.name); + const relPath = path.relative(absExtDir, absPath); + if (relPath !== MANIFEST_FILE_NAME) { + zipFile.addFile(absPath, relPath, { + compress: true, + compressionLevel: 1, + }); + } + } + } + + await new Promise((resolve, reject) => { + zipFile.outputStream.once('error', reject); + zipFile.outputStream + .pipe(fs.createWriteStream(xpiPath)) + .once('error', reject) + .once('close', () => resolve()); + zipFile.end({ comment: manifestHash, forceZip64Format: false }); + }); + + return xpiPath; + } + + async function measureInstallAddon(xpiPath: string) { + const options = new firefox.Options(); + options.addArguments('-headless'); + options.setAcceptInsecureCerts(true); + const driver = await new Builder() + .forBrowser('firefox') + .setFirefoxOptions(options) + .build(); + + try { + const start = performance.now(); + await driver.installAddon(xpiPath, true); + return performance.now() - start; + } finally { + await driver.quit(); + } + } + + const variants: Record = { + zip1: buildZip1Xpi(), + yazlCurrent: await buildYazlXpi('yazl-current', paddedManifest, { + compress: false, + }), + yazlManifestCompressed: await buildYazlXpi( + 'yazl-manifest-compressed', + rawManifest, + { compress: true, compressionLevel: 1 }, + ), + yazlManifestStoredRaw: await buildYazlXpi( + 'yazl-manifest-stored-raw', + rawManifest, + { compress: false }, + ), + }; + + const results = {} as Record; + + for (const [name, xpiPath] of Object.entries(variants) as [ + VariantName, + string, + ][]) { + const samples: number[] = []; + for (let iteration = 0; iteration < iterations; iteration += 1) { + samples.push(await measureInstallAddon(xpiPath)); + } + + results[name] = { + path: xpiPath, + size: fs.statSync(xpiPath).size, + samples, + summary: summarize(samples), + }; + } + + const gitSha = requireFromTarget('node:child_process') + .execFileSync('git', ['rev-parse', 'HEAD'], { cwd: root }) + .toString() + .trim(); + + const result: BenchmarkResult = { + gitRef: refLabel, + gitSha, + iterations, + variants: results, + }; + + console.log( + `Firefox installAddon variant benchmark for ${refLabel} (${gitSha})`, + ); + console.log( + 'Variant'.padEnd(28) + + 'size'.padStart(14) + + 'mean'.padStart(14) + + 'median'.padStart(14) + + 'p95'.padStart(14), + ); + + for (const [name, variant] of Object.entries(result.variants) as [ + VariantName, + VariantResult, + ][]) { + console.log( + name.padEnd(28) + + `${(variant.size / (1024 * 1024)).toFixed(1)} MiB`.padStart(14) + + formatMs(variant.summary.mean).padStart(14) + + formatMs(variant.summary.median).padStart(14) + + formatMs(variant.summary.p95).padStart(14), + ); + } + + if (out) { + fs.mkdirSync(path.dirname(out), { recursive: true }); + fs.writeFileSync(out, JSON.stringify(result, null, 2)); + } +} + +main().catch((error: unknown) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/.github/workflows/firefox-buildwebdriver-benchmark-temp.yml b/.github/workflows/firefox-buildwebdriver-benchmark-temp.yml new file mode 100644 index 000000000000..499e78b11790 --- /dev/null +++ b/.github/workflows/firefox-buildwebdriver-benchmark-temp.yml @@ -0,0 +1,173 @@ +name: Firefox buildWebDriver benchmark (temp) + +on: + workflow_dispatch: + inputs: + iterations: + description: Benchmark iterations per scenario + required: false + default: '7' + pull_request: + paths: + - .github/workflows/firefox-buildwebdriver-benchmark-temp.yml + - .github/scripts/firefox-buildwebdriver-benchmark.ts + +permissions: + contents: read + +env: + CANDIDATE_REF: codex/firefox-xpi-comment-metadata-spike + +jobs: + benchmark: + runs-on: ubuntu-latest + timeout-minutes: 90 + container: + image: ghcr.io/metamask/metamask-extension-e2e-image:v24.13.0 + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + strategy: + fail-fast: false + matrix: + include: + - name: main + ref: main + - name: candidate + ref: codex/firefox-xpi-comment-metadata-spike + name: benchmark-${{ matrix.name }} + env: + HEADLESS: false + INFURA_PROJECT_ID: ${{ secrets.INFURA_PROJECT_ID }} + ITERATIONS: ${{ inputs.iterations || '7' }} + SELENIUM_BROWSER: firefox + steps: + - name: Checkout harness branch + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Copy benchmark script to temp + run: cp .github/scripts/firefox-buildwebdriver-benchmark.ts /tmp/ + + - name: Checkout target ref + uses: actions/checkout@v4 + with: + ref: ${{ matrix.ref }} + clean: true + persist-credentials: false + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version-file: .nvmrc + cache: yarn + + - name: Install dependencies + run: | + corepack enable + yarn install --immutable + + - name: Create .metamaskrc + run: | + cp .metamaskrc.dist .metamaskrc + echo "INFURA_PROJECT_ID=${INFURA_PROJECT_ID}" >> .metamaskrc + + - name: Configure Xvfb + run: Xvfb -ac :99 -screen 0 1280x1024x16 & + + - name: Build Firefox test bundle + run: yarn build:test:mv2 + + - name: Run buildWebDriver benchmark + env: + DISPLAY: :99 + run: | + mkdir -p test-artifacts/benchmarks + yarn tsx /tmp/firefox-buildwebdriver-benchmark.ts \ + --iterations "${ITERATIONS}" \ + --ref-label "${{ matrix.name }}" \ + --out "test-artifacts/benchmarks/firefox-buildwebdriver-${{ matrix.name }}.json" + + - name: Upload benchmark artifact + uses: actions/upload-artifact@v4 + with: + name: firefox-buildwebdriver-${{ matrix.name }} + path: test-artifacts/benchmarks/firefox-buildwebdriver-${{ matrix.name }}.json + retention-days: 7 + + compare: + runs-on: ubuntu-latest + needs: benchmark + if: ${{ always() }} + steps: + - name: Download benchmark artifacts + uses: actions/download-artifact@v4 + with: + pattern: firefox-buildwebdriver-* + path: benchmark-results + merge-multiple: true + + - name: Compare results + run: | + node <<'EOF' >> "$GITHUB_STEP_SUMMARY" + const fs = require('fs'); + const path = require('path'); + + const main = JSON.parse( + fs.readFileSync( + path.join('benchmark-results', 'firefox-buildwebdriver-main.json'), + 'utf8', + ), + ); + const candidate = JSON.parse( + fs.readFileSync( + path.join( + 'benchmark-results', + 'firefox-buildwebdriver-candidate.json', + ), + 'utf8', + ), + ); + + const scenarios = ['cold', 'warmUnchanged', 'warmManifestUpdate']; + const metrics = ['mean', 'median', 'p95']; + const phases = [ + ['builderBuildMs', 'builder.build'], + ['addonSetupMs', 'addonSetup'], + ['installExtensionMs', 'installExtension'], + ['getInternalIdMs', 'getInternalId'], + ['totalMs', 'total'], + ]; + + console.log('# Firefox buildWebDriver benchmark'); + console.log(''); + console.log(`Main SHA: \`${main.gitSha}\``); + console.log(`Candidate SHA: \`${candidate.gitSha}\``); + console.log(''); + + for (const scenario of scenarios) { + console.log(`## ${scenario}`); + console.log(''); + for (const metric of metrics) { + console.log(`### ${metric}`); + console.log(''); + console.log('| Phase | main | candidate | delta |'); + console.log('| --- | ---: | ---: | ---: |'); + for (const [key, label] of phases) { + const mainValue = main.scenarios[scenario].summary[key][metric]; + const candidateValue = + candidate.scenarios[scenario].summary[key][metric]; + const delta = candidateValue - mainValue; + const formattedDelta = + `${delta >= 0 ? '+' : ''}${delta.toFixed(1)} ms`; + console.log( + `| ${label} | ${mainValue.toFixed(1)} ms | ${candidateValue.toFixed( + 1, + )} ms | ${formattedDelta} |`, + ); + } + console.log(''); + } + } + EOF diff --git a/.github/workflows/firefox-installaddon-variant-benchmark-temp.yml b/.github/workflows/firefox-installaddon-variant-benchmark-temp.yml new file mode 100644 index 000000000000..4a741d5477d8 --- /dev/null +++ b/.github/workflows/firefox-installaddon-variant-benchmark-temp.yml @@ -0,0 +1,115 @@ +name: Firefox installAddon variant benchmark (temp) + +on: + workflow_dispatch: + inputs: + iterations: + description: Benchmark iterations per variant + required: false + default: '7' + target_ref: + description: Ref to build before benchmarking variants + required: false + default: codex/firefox-xpi-comment-metadata-spike + pull_request: + paths: + - .github/workflows/firefox-installaddon-variant-benchmark-temp.yml + - .github/scripts/firefox-installaddon-variant-benchmark.ts + +permissions: + contents: read + +jobs: + benchmark: + runs-on: ubuntu-latest + timeout-minutes: 90 + container: + image: ghcr.io/metamask/metamask-extension-e2e-image:v24.13.0 + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + env: + HEADLESS: false + INFURA_PROJECT_ID: ${{ secrets.INFURA_PROJECT_ID }} + ITERATIONS: ${{ inputs.iterations || '7' }} + SELENIUM_BROWSER: firefox + TARGET_REF: ${{ inputs.target_ref || 'codex/firefox-xpi-comment-metadata-spike' }} + steps: + - name: Checkout harness branch + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Copy benchmark script to temp + run: cp .github/scripts/firefox-installaddon-variant-benchmark.ts /tmp/ + + - name: Checkout target ref + uses: actions/checkout@v4 + with: + ref: ${{ env.TARGET_REF }} + clean: true + persist-credentials: false + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version-file: .nvmrc + cache: yarn + + - name: Install dependencies + run: | + corepack enable + yarn install --immutable + + - name: Create .metamaskrc + run: | + cp .metamaskrc.dist .metamaskrc + echo "INFURA_PROJECT_ID=${INFURA_PROJECT_ID}" >> .metamaskrc + + - name: Configure Xvfb + run: Xvfb -ac :99 -screen 0 1280x1024x16 & + + - name: Build Firefox test bundle + run: yarn build:test:mv2 + + - name: Run installAddon variant benchmark + env: + DISPLAY: :99 + run: | + mkdir -p test-artifacts/benchmarks + yarn tsx /tmp/firefox-installaddon-variant-benchmark.ts \ + --iterations "${ITERATIONS}" \ + --ref-label "${TARGET_REF}" \ + --out "test-artifacts/benchmarks/firefox-installaddon-variant-benchmark.json" + + - name: Upload benchmark artifact + uses: actions/upload-artifact@v4 + with: + name: firefox-installaddon-variant-benchmark + path: test-artifacts/benchmarks/firefox-installaddon-variant-benchmark.json + retention-days: 7 + + - name: Write step summary + run: | + node <<'EOF' >> "$GITHUB_STEP_SUMMARY" + const fs = require('fs'); + const result = JSON.parse( + fs.readFileSync( + 'test-artifacts/benchmarks/firefox-installaddon-variant-benchmark.json', + 'utf8', + ), + ); + + console.log('# Firefox installAddon variant benchmark'); + console.log(''); + console.log(`Target ref: \`${result.gitRef}\``); + console.log(`SHA: \`${result.gitSha}\``); + console.log(''); + console.log('| Variant | Size | Mean | Median | P95 |'); + console.log('| --- | ---: | ---: | ---: | ---: |'); + for (const [name, variant] of Object.entries(result.variants)) { + console.log( + `| ${name} | ${(variant.size / (1024 * 1024)).toFixed(1)} MiB | ${variant.summary.mean.toFixed(1)} ms | ${variant.summary.median.toFixed(1)} ms | ${variant.summary.p95.toFixed(1)} ms |`, + ); + } + EOF