chore: compare unlighthouse reports preview vs production

Author: jorgemoya

.github/scripts/compare-unlighthouse.mts

@@ -0,0 +1,335 @@
+#!/usr/bin/env node
+/* eslint-disable no-console, no-restricted-syntax, no-plusplus, no-continue */
+
+import { existsSync, readFileSync, writeFileSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+import { parseArgs } from "node:util";
+import { resolve } from "node:path";
+
+interface CategoryScore {
+  averageScore: number;
+}
+
+interface MetricValue {
+  averageNumericValue: number;
+}
+
+interface CiResult {
+  summary: {
+    categories: Record<string, CategoryScore>;
+    metrics: Record<string, MetricValue>;
+  };
+}
+
+interface DeviceResult {
+  categories: Record<string, number>;
+  metrics: Record<string, number>;
+}
+
+interface DeviceComparison {
+  categoryRows: string[];
+  metricRows: string[];
+  changed: boolean;
+}
+
+function loadCiResult(filePath: string): CiResult {
+  if (!existsSync(filePath)) {
+    console.error(`Error: file not found: ${filePath}`);
+    process.exit(1);
+  }
+
+  return JSON.parse(readFileSync(filePath, "utf-8")) as CiResult;
+}
+
+function extractDevice(ci: CiResult): DeviceResult {
+  const categories: Record<string, number> = {};
+
+  for (const id of CATEGORY_ORDER) {
+    const score = ci.summary.categories[id]?.averageScore;
+
+    if (score !== undefined) categories[id] = score;
+  }
+
+  const metrics: Record<string, number> = {};
+
+  for (const id of METRIC_ORDER) {
+    const val = ci.summary.metrics[id]?.averageNumericValue;
+
+    if (val !== undefined) metrics[id] = val;
+  }
+
+  return { categories, metrics };
+}
+
+function formatScore(score: number): string {
+  return `${Math.round(score * 100)}%`;
+}
+
+function formatDeltaScore(delta: number): string {
+  const pp = Math.round(delta * 100);
+  const sign = pp >= 0 ? "+" : "";
+  const arrow = pp > 0 ? "↑" : "↓";
+
+  return `${arrow} ${sign}${pp}pp`;
+}
+
+function formatTime(ms: number): string {
+  if (ms >= 1000) return `${(ms / 1000).toFixed(2)}s`;
+
+  return `${Math.round(ms)}ms`;
+}
+
+function formatDeltaTime(delta: number): string {
+  // For time metrics, lower is better: negative delta = improvement (↑)
+  const sign = delta >= 0 ? "+" : "";
+  const arrow = delta < 0 ? "↑" : "↓";
+
+  return `${arrow} ${sign}${formatTime(delta)}`;
+}
+
+function formatCls(value: number): string {
+  return value.toFixed(3);
+}
+
+function formatDeltaCls(delta: number): string {
+  // For CLS, lower is better: negative delta = improvement (↑)
+  const sign = delta >= 0 ? "+" : "";
+  const arrow = delta < 0 ? "↑" : "↓";
+
+  return `${arrow} ${sign}${delta.toFixed(3)}`;
+}
+
+const CATEGORY_ORDER = ["performance", "accessibility", "best-practices", "seo"];
+
+const CATEGORY_LABELS: Record<string, string> = {
+  performance: "Performance",
+  accessibility: "Accessibility",
+  "best-practices": "Best Practices",
+  seo: "SEO",
+};
+
+const METRIC_ORDER = [
+  "largest-contentful-paint",
+  "first-contentful-paint",
+  "total-blocking-time",
+  "cumulative-layout-shift",
+];
+
+const METRIC_LABELS: Record<string, string> = {
+  "largest-contentful-paint": "LCP",
+  "first-contentful-paint": "FCP",
+  "total-blocking-time": "TBT",
+  "cumulative-layout-shift": "CLS",
+};
+
+function compareDevice(
+  production: DeviceResult,
+  preview: DeviceResult,
+  threshold: number,
+): DeviceComparison {
+  let changed = false;
+  const categoryRows: string[] = [];
+  const metricRows: string[] = [];
+
+  for (const id of CATEGORY_ORDER) {
+    const prod = production.categories[id];
+    const prev = preview.categories[id];
+
+    if (prod === undefined || prev === undefined) continue;
+
+    const delta = prev - prod;
+
+    if (Math.abs(Math.round(delta * 100)) < threshold) continue;
+
+    changed = true;
+    categoryRows.push(
+      `| ${CATEGORY_LABELS[id] ?? id} | ${formatScore(prod)} | ${formatScore(prev)} | ${formatDeltaScore(delta)} |`,
+    );
+  }
+
+  for (const id of METRIC_ORDER) {
+    const prod = production.metrics[id];
+    const prev = preview.metrics[id];
+
+    if (prod === undefined || prev === undefined) continue;
+
+    const delta = prev - prod;
+
+    if (id === "cumulative-layout-shift") {
+      if (Math.abs(delta) < 0.01) continue;
+      changed = true;
+      metricRows.push(
+        `| ${METRIC_LABELS[id]} | ${formatCls(prod)} | ${formatCls(prev)} | ${formatDeltaCls(delta)} |`,
+      );
+    } else {
+      if (Math.abs(delta) < threshold * 50) continue;
+      changed = true;
+      metricRows.push(
+        `| ${METRIC_LABELS[id]} | ${formatTime(prod)} | ${formatTime(prev)} | ${formatDeltaTime(delta)} |`,
+      );
+    }
+  }
+
+  return { categoryRows, metricRows, changed };
+}
+
+function deviceSection(
+  label: string,
+  rows: string[],
+  header: string,
+  separator: string,
+): string[] {
+  const lines = [`**${label}**`, ""];
+
+  if (rows.length > 0) {
+    lines.push(header, separator, ...rows);
+  } else {
+    lines.push("_No changes detected._");
+  }
+
+  lines.push("");
+
+  return lines;
+}
+
+function compareResults(
+  productionDesktop: CiResult,
+  productionMobile: CiResult,
+  previewDesktop: CiResult,
+  previewMobile: CiResult,
+  threshold: number,
+  provider?: string,
+): { markdown: string; hasChanges: boolean } {
+  const desktop = compareDevice(
+    extractDevice(productionDesktop),
+    extractDevice(previewDesktop),
+    threshold,
+  );
+  const mobile = compareDevice(
+    extractDevice(productionMobile),
+    extractDevice(previewMobile),
+    threshold,
+  );
+
+  const hasChanges = desktop.changed || mobile.changed;
+  const lines: string[] = [];
+
+  const providerLabel = provider
+    ? ` — ${provider.charAt(0).toUpperCase()}${provider.slice(1)}`
+    : "";
+
+  lines.push(`## Unlighthouse Performance Comparison${providerLabel}`);
+  lines.push(
+    `Comparing PR preview against production. Changes ≥ ${threshold}pp shown.`,
+  );
+  lines.push("> ↑ = improved · ↓ = regressed");
+  lines.push("");
+
+  lines.push("### Category Scores");
+  lines.push("");
+  lines.push(
+    ...deviceSection(
+      "Desktop",
+      desktop.categoryRows,
+      "| Category | Production | PR Preview | Delta |",
+      "|:---------|:-----------|:-----------|:------|",
+    ),
+  );
+  lines.push(
+    ...deviceSection(
+      "Mobile",
+      mobile.categoryRows,
+      "| Category | Production | PR Preview | Delta |",
+      "|:---------|:-----------|:-----------|:------|",
+    ),
+  );
+
+  lines.push("### Core Web Vitals");
+  lines.push("");
+  lines.push(
+    ...deviceSection(
+      "Desktop",
+      desktop.metricRows,
+      "| Metric | Production | PR Preview | Delta |",
+      "|:-------|:-----------|:-----------|:------|",
+    ),
+  );
+  lines.push(
+    ...deviceSection(
+      "Mobile",
+      mobile.metricRows,
+      "| Metric | Production | PR Preview | Delta |",
+      "|:-------|:-----------|:-----------|:------|",
+    ),
+  );
+
+  return { markdown: lines.join("\n"), hasChanges };
+}
+
+export { extractDevice, compareDevice, compareResults };
+export type { CiResult, DeviceResult, DeviceComparison };
+
+const isMain = process.argv[1] === fileURLToPath(import.meta.url);
+
+if (isMain) {
+  const { values } = parseArgs({
+    options: {
+      "preview-desktop": { type: "string" },
+      "preview-mobile": { type: "string" },
+      "production-desktop": { type: "string" },
+      "production-mobile": { type: "string" },
+      output: { type: "string" },
+      "meta-output": { type: "string" },
+      threshold: { type: "string" },
+      provider: { type: "string" },
+    },
+  });
+
+  const previewDesktopPath = values["preview-desktop"] ?? "";
+  const previewMobilePath = values["preview-mobile"] ?? "";
+  const productionDesktopPath = values["production-desktop"] ?? "";
+  const productionMobilePath = values["production-mobile"] ?? "";
+
+  if (
+    !previewDesktopPath ||
+    !previewMobilePath ||
+    !productionDesktopPath ||
+    !productionMobilePath
+  ) {
+    console.error(
+      "Usage: compare-unlighthouse.mts --preview-desktop <path> --preview-mobile <path> --production-desktop <path> --production-mobile <path> [--output <path>] [--meta-output <path>] [--threshold <n>] [--provider <name>]",
+    );
+    process.exit(1);
+  }
+
+  const threshold = Number(values.threshold ?? "1");
+
+  const previewDesktop = loadCiResult(resolve(previewDesktopPath));
+  const previewMobile = loadCiResult(resolve(previewMobilePath));
+  const productionDesktop = loadCiResult(resolve(productionDesktopPath));
+  const productionMobile = loadCiResult(resolve(productionMobilePath));
+
+  const { markdown, hasChanges } = compareResults(
+    productionDesktop,
+    productionMobile,
+    previewDesktop,
+    previewMobile,
+    threshold,
+    values.provider,
+  );
+
+  const outputPath = values.output ? resolve(values.output) : null;
+  const metaOutputPath = values["meta-output"] ? resolve(values["meta-output"]) : null;
+
+  if (outputPath) {
+    writeFileSync(outputPath, markdown);
+    console.error(`Unlighthouse comparison report written to ${outputPath}`);
+  } else {
+    process.stdout.write(markdown);
+  }
+
+  if (metaOutputPath) {
+    writeFileSync(metaOutputPath, `${JSON.stringify({ hasChanges }, null, 2)}\n`);
+    console.error(`Meta output written to ${metaOutputPath}`);
+  }
+}

.github/scripts/post-unlighthouse-comment.js

@@ -0,0 +1,52 @@
+const fs = require('fs');
+
+module.exports = async ({ github, context, provider = 'unknown', reportPath = '/tmp/unlighthouse-report.md', metaPath = '/tmp/unlighthouse-meta.json' }) => {
+  // Exit early if no changes
+  const { hasChanges } = JSON.parse(fs.readFileSync(metaPath, 'utf-8'));
+
+  if (!hasChanges) return;
+
+  // Find PR from commit SHA (context.issue.number is 0 in deployment_status events;
+  // deployment.ref is also the SHA in Vercel deployments, not the branch name)
+  const sha = context.payload.deployment?.sha;
+
+  if (!sha) return;
+
+  const { data: prs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({
+    owner: context.repo.owner,
+    repo: context.repo.repo,
+    commit_sha: sha,
+  });
+
+  const prNumber = prs[0]?.number;
+
+  if (!prNumber) return;
+
+  const runUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
+  const marker = `<!-- unlighthouse-${provider}-report -->`;
+  const body = marker + '\n' + fs.readFileSync(reportPath, 'utf-8') + `[Full Unlighthouse report →](${runUrl})\n`;
+
+  const { data: comments } = await github.rest.issues.listComments({
+    owner: context.repo.owner,
+    repo: context.repo.repo,
+    issue_number: prNumber,
+  });
+
+  const existing = comments.find(c => c.body.includes(marker));
+
+  if (existing) {
+    await github.rest.issues.updateComment({
+      owner: context.repo.owner,
+      repo: context.repo.repo,
+      comment_id: existing.id,
+      body,
+    });
+  } else {
+    await github.rest.issues.createComment({
+      owner: context.repo.owner,
+      repo: context.repo.repo,
+      issue_number: prNumber,
+      body,
+    });
+  }
+};