diff --git a/torchci/components/benchmark_v3/configs/teams/compilers/BetterBenchmarkSummary.tsx b/torchci/components/benchmark_v3/configs/teams/compilers/BetterBenchmarkSummary.tsx
new file mode 100644
index 0000000000..0f4d234c4b
--- /dev/null
+++ b/torchci/components/benchmark_v3/configs/teams/compilers/BetterBenchmarkSummary.tsx
@@ -0,0 +1,864 @@
+import InfoOutlinedIcon from "@mui/icons-material/InfoOutlined";
+import TrendingDownIcon from "@mui/icons-material/TrendingDown";
+import TrendingFlatIcon from "@mui/icons-material/TrendingFlat";
+import TrendingUpIcon from "@mui/icons-material/TrendingUp";
+import {
+ Alert,
+ Box,
+ Card,
+ CardContent,
+ Chip,
+ Divider,
+ IconButton,
+ Paper,
+ Stack,
+ Tab,
+ Tabs,
+ Tooltip,
+ Typography,
+} from "@mui/material";
+import { DataGrid, GridColDef, GridPaginationModel } from "@mui/x-data-grid";
+import type { AutoComponentProps } from "components/benchmark_v3/configs/utils/autoRegistration";
+import LoadingPage from "components/common/LoadingPage";
+import type {
+ BetterBenchmarkSummaryData,
+ RollupStats,
+} from "lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/betterBenchmarkSummary";
+import {
+ useBenchmarkCommittedContext,
+ useBenchmarkTimeSeriesData,
+} from "lib/benchmark/api_helper/fe/hooks";
+import { useMemo, useState } from "react";
+
+type Mover = BetterBenchmarkSummaryData["models"][number];
+type MoverTab = "gains" | "losses" | "neutral";
+
+const NEUTRAL_THRESHOLD_PCT = 5;
+
+function moverCategory(value: number): MoverTab {
+ if (value >= NEUTRAL_THRESHOLD_PCT) {
+ return "gains";
+ }
+ if (value <= -NEUTRAL_THRESHOLD_PCT) {
+ return "losses";
+ }
+ return "neutral";
+}
+
+function moverCounts(source: Mover[]) {
+ return source.reduce(
+ (counts, row) => {
+ counts[moverCategory(row.reductionPct)] += 1;
+ return counts;
+ },
+ { gains: 0, losses: 0, neutral: 0 }
+ );
+}
+
+function speedupPctToReductionPct(value: number) {
+ return (1 - 1 / (1 + value / 100)) * 100;
+}
+
+function formatDelta(value: number, digits = 2) {
+ return `${value >= 0 ? "+" : ""}${value.toFixed(digits)}%`;
+}
+
+function formatLatency(value: number, signed = false) {
+ const prefix = signed && value >= 0.005 ? "+" : "";
+ const absolute = Math.abs(value);
+ if (absolute >= 1000) {
+ return `${prefix}${(value / 1000).toLocaleString(undefined, {
+ minimumFractionDigits: 2,
+ maximumFractionDigits: 2,
+ })} ms`;
+ }
+ return `${prefix}${value.toLocaleString(undefined, {
+ minimumFractionDigits: 2,
+ maximumFractionDigits: 2,
+ })} µs`;
+}
+
+function DeltaValue({
+ value,
+ large = false,
+ speedup = false,
+}: {
+ value: number;
+ large?: boolean;
+ speedup?: boolean;
+}) {
+ const classifiedValue = speedup ? speedupPctToReductionPct(value) : value;
+ const neutral = Math.abs(classifiedValue) < NEUTRAL_THRESHOLD_PCT;
+ const improving = classifiedValue >= NEUTRAL_THRESHOLD_PCT;
+ return (
+
+ {neutral ? (
+
+ ) : improving ? (
+
+ ) : (
+
+ )}
+
+ {formatDelta(value, large ? 3 : 2)}
+
+
+ );
+}
+
+function LatencySavedValue({ value }: { value: number }) {
+ return (
+ 0
+ ? "success.main"
+ : value < 0
+ ? "error.main"
+ : "text.secondary",
+ fontVariantNumeric: "tabular-nums",
+ }}
+ >
+ {formatLatency(value, true)}
+
+ );
+}
+
+function SummaryCard({
+ title,
+ description,
+ stats,
+ projected = false,
+ unavailableReason = "",
+ distribution,
+}: {
+ title: string;
+ description: string;
+ stats: RollupStats | null;
+ projected?: boolean;
+ unavailableReason?: string;
+ distribution?: ReturnType;
+}) {
+ return (
+
+
+
+
+
+
+ {title}
+
+
+
+
+
+
+
+ {projected && (
+
+ Projected end-to-end estimate
+
+ )}
+
+
+
+ {stats ? (
+ <>
+
+
+ Geometric-mean speedup
+
+
+
+ Median and mean below are per-item latency changes. Positive is
+ faster; negative is slower.
+
+ {distribution && (
+
+ {distribution.gains} improved · {distribution.losses}{" "}
+ regressed · {distribution.neutral} neutral
+
+ )}
+
+
+ }
+ >
+
+
+ Median latency change
+
+
+ {formatDelta(stats.median, 3)}
+
+
+
+
+ Mean latency change
+
+
+ {formatDelta(stats.mean, 3)}
+
+
+
+
+ ) : (
+
+ {unavailableReason ||
+ "No exact points are shared by the selected workflows."}
+
+ )}
+
+
+ );
+}
+
+function SectionHeader({
+ title,
+ description,
+}: {
+ title: string;
+ description: string;
+}) {
+ return (
+
+
+ {title}
+
+
+ {description}
+
+
+ );
+}
+
+function SuiteBreakdown({
+ rows,
+}: {
+ rows: BetterBenchmarkSummaryData["suites"];
+}) {
+ const max = Math.max(
+ 0,
+ ...rows.map((row) => Math.abs(row.stats?.geomean ?? 0))
+ );
+ const domain = Math.max(2, Math.ceil(max));
+ return (
+
+
+ {rows.length === 0 ? (
+
+ No complete model comparisons.
+
+ ) : (
+
+ {rows.map((row) => (
+
+ {row.suiteMode}
+
+
+
+ 0
+ ? "success.main"
+ : "error.main",
+ }}
+ />
+
+
+
+
+ Median
+
+
+ {row.stats ? formatDelta(row.stats.median, 3) : "n/a"}
+
+
+
+
+ Mean
+
+
+ {row.stats ? formatDelta(row.stats.mean, 3) : "n/a"}
+
+
+
+ {row.stats ? (
+
+ ) : (
+ n/a
+ )}
+
+ n={row.stats?.n ?? 0}
+
+
+
+ ))}
+
+ )}
+
+ );
+}
+
+const latencyColumn = (
+ field: "baseUs" | "headUs" | "deltaUs",
+ headerName: string,
+ signed = false
+): GridColDef =>
+ signed
+ ? {
+ field,
+ headerName,
+ type: "number",
+ display: "flex",
+ minWidth: 140,
+ renderCell: (params) => (
+
+ ),
+ }
+ : {
+ field,
+ headerName,
+ type: "number",
+ minWidth: 120,
+ valueFormatter: (value) => formatLatency(Number(value)),
+ };
+
+const modelColumns: GridColDef[] = [
+ {
+ field: "name",
+ headerName: "Model",
+ minWidth: 270,
+ flex: 1,
+ renderCell: (params) => (
+
+ {params.value as string}
+
+ ),
+ },
+ { field: "suite", headerName: "Suite", minWidth: 115 },
+ { field: "mode", headerName: "Mode", minWidth: 100 },
+ latencyColumn("baseUs", "Baseline"),
+ latencyColumn("headUs", "Candidate"),
+ latencyColumn("deltaUs", "Latency saved", true),
+ {
+ field: "reductionPct",
+ headerName: "Projected change",
+ type: "number",
+ display: "flex",
+ minWidth: 160,
+ renderCell: (params) => ,
+ },
+];
+
+const kernelColumns: GridColDef[] = [
+ {
+ field: "name",
+ headerName: "Kernel shape",
+ minWidth: 245,
+ flex: 1,
+ renderCell: (params) => (
+
+ {params.value as string}
+
+ ),
+ },
+ {
+ field: "exampleModel",
+ headerName: "Example model",
+ minWidth: 220,
+ flex: 0.8,
+ },
+ latencyColumn("baseUs", "Baseline"),
+ latencyColumn("headUs", "Candidate"),
+ latencyColumn("deltaUs", "Latency saved", true),
+ {
+ field: "reductionPct",
+ headerName: "Change",
+ type: "number",
+ display: "flex",
+ minWidth: 130,
+ renderHeader: () => (
+
+
+
+ Change
+
+
+
+
+ ),
+ renderCell: (params) => ,
+ },
+ {
+ field: "headGapVsSol",
+ headerName: "Gap vs SOL",
+ type: "number",
+ minWidth: 175,
+ renderHeader: () => (
+
+
+
+ Gap vs SOL
+
+
+
+
+ ),
+ renderCell: (params) => {
+ const baseline = params.row.baseGapVsSol;
+ const candidate = params.row.headGapVsSol;
+ if (baseline == null || candidate == null) {
+ return n/a;
+ }
+ return (
+
+
+ {baseline.toFixed(2)}× → {candidate.toFixed(2)}×
+
+
+ );
+ },
+ },
+];
+
+function MoversTable({
+ kind,
+ source,
+}: {
+ kind: "models" | "kernels";
+ source: Mover[];
+}) {
+ const [tab, setTab] = useState("gains");
+ const [paginationModel, setPaginationModel] = useState({
+ pageSize: 10,
+ page: 0,
+ });
+ const counts = useMemo(() => moverCounts(source), [source]);
+ const rows = useMemo(
+ () =>
+ source
+ .filter((row) => moverCategory(row.reductionPct) === tab)
+ .sort((a, b) => {
+ if (tab === "gains") {
+ return b.deltaUs - a.deltaUs;
+ }
+ if (tab === "losses") {
+ return a.deltaUs - b.deltaUs;
+ }
+ return Math.abs(b.reductionPct) - Math.abs(a.reductionPct);
+ }),
+ [source, tab]
+ );
+
+ return (
+
+
+ {
+ setTab(value);
+ setPaginationModel((current) => ({ ...current, page: 0 }));
+ }}
+ sx={{ mb: 1 }}
+ >
+
+ Gains ({counts.gains})
+
+ }
+ />
+
+ Losses ({counts.losses})
+
+ }
+ />
+
+ Neutral ({counts.neutral})
+
+ }
+ />
+
+
+
+ );
+}
+
+function BetterBenchmarkSummary({
+ data,
+ repo,
+}: {
+ data: BetterBenchmarkSummaryData;
+ repo: string;
+}) {
+ const workflowUrl = (workflow: string) =>
+ `https://github.com/${repo}/actions/runs/${workflow}`;
+ const hasRunQualityWarnings = [
+ data.coverage.baselineRun,
+ data.coverage.candidateRun,
+ ].some((run) =>
+ [
+ run.failedRepros,
+ run.invalidMeasurements,
+ run.missingShapeFiles,
+ run.unresolvedShapeMetadata,
+ ].some((value) => (value ?? 0) > 0)
+ );
+ const hasUnknownRunQuality =
+ !data.coverage.baselineRun.available ||
+ !data.coverage.candidateRun.available;
+
+ return (
+
+ {data.comparisonUnavailableReason && (
+
+ {data.comparisonUnavailableReason}
+
+ )}
+ {data.modelUnavailableReason &&
+ data.modelUnavailableReason !== data.comparisonUnavailableReason && (
+
+ {data.modelUnavailableReason}. Kernel results remain comparable.
+
+ )}
+
+
+ Summary comparison
+
+
+
+ →
+
+
+
+
+ {data.coverage.matchedKernelPoints.toLocaleString()} matched kernel
+ points
+ {data.coverage.leftOnlyKernelPoints > 0
+ ? ` · ${data.coverage.leftOnlyKernelPoints.toLocaleString()} baseline-only`
+ : ""}
+ {data.coverage.rightOnlyKernelPoints > 0
+ ? ` · ${data.coverage.rightOnlyKernelPoints.toLocaleString()} candidate-only`
+ : ""}
+ {data.coverage.invalidBaselineKernelPoints > 0
+ ? ` · ${data.coverage.invalidBaselineKernelPoints.toLocaleString()} invalid baseline`
+ : ""}
+ {data.coverage.invalidCandidateKernelPoints > 0
+ ? ` · ${data.coverage.invalidCandidateKernelPoints.toLocaleString()} invalid candidate`
+ : ""}
+ {data.coverage.incompatibleModels > 0
+ ? ` · ${data.coverage.incompatibleModels.toLocaleString()} accounting-incompatible models`
+ : ""}
+ {" · "}
+ {data.coverage.includedModels}/{data.coverage.totalModels} paired
+ complete models
+
+
+ {hasUnknownRunQuality && (
+
+ Sweep-quality metadata is unavailable for one or both workflows.
+
+ )}
+ {hasRunQualityWarnings && (
+
+ Partial sweep data detected. Baseline:{" "}
+ {data.coverage.baselineRun.failedRepros ?? "unknown"} failed repros,{" "}
+ {data.coverage.baselineRun.invalidMeasurements ?? "unknown"} invalid
+ measurements,{" "}
+ {data.coverage.baselineRun.missingShapeFiles ?? "unknown"} missing
+ shape files,{" "}
+ {data.coverage.baselineRun.unresolvedShapeMetadata ?? "unknown"}{" "}
+ unresolved shape points; candidate:{" "}
+ {data.coverage.candidateRun.failedRepros ?? "unknown"} failed repros,{" "}
+ {data.coverage.candidateRun.invalidMeasurements ?? "unknown"} invalid
+ measurements,{" "}
+ {data.coverage.candidateRun.missingShapeFiles ?? "unknown"} missing
+ shape files,{" "}
+ {data.coverage.candidateRun.unresolvedShapeMetadata ?? "unknown"}{" "}
+ unresolved shape points. Metrics include only valid exported points.
+
+ )}
+
+
+
+
+
+
+
+ Changes smaller than {NEUTRAL_THRESHOLD_PCT}% are shown as neutral.
+
+
+
+
+
+
+ );
+}
+
+export function AutoBetterBenchmarkSummary({ config }: AutoComponentProps) {
+ const ctx = useBenchmarkCommittedContext();
+ const leftWorkflow = ctx.lcommit?.workflow_id;
+ const rightWorkflow = ctx.rcommit?.workflow_id;
+ const ready =
+ !!ctx.committedTime?.start &&
+ !!ctx.committedTime?.end &&
+ leftWorkflow != null &&
+ rightWorkflow != null;
+
+ const params = ctx.configHandler.dataBinding.toQueryParams({
+ repo: ctx.repo,
+ branches: [
+ ...new Set([ctx.committedLbranch, ctx.committedRbranch].filter(Boolean)),
+ ],
+ workflows: [String(leftWorkflow ?? ""), String(rightWorkflow ?? "")],
+ benchmarkName: ctx.benchmarkName,
+ timeRange: ctx.committedTime,
+ filters: ctx.committedFilters,
+ maxSampling: ctx.committedMaxSampling,
+ });
+ const fetcherId = config?.config?.fetcherId ?? ctx.benchmarkId;
+ const {
+ data: response,
+ isLoading,
+ error,
+ } = useBenchmarkTimeSeriesData(fetcherId, ready ? params : null, [
+ "better_summary",
+ ]);
+
+ if (!ready) {
+ return (
+
+ Select both comparison commits to load the performance summary.
+
+ );
+ }
+ if (isLoading) {
+ return (
+
+ );
+ }
+ if (error) {
+ return {error.message};
+ }
+
+ const summary = response?.data?.data?.better_summary as
+ | BetterBenchmarkSummaryData
+ | undefined;
+ if (!summary) {
+ return No summary data found.;
+ }
+ return ;
+}
diff --git a/torchci/components/benchmark_v3/configs/teams/compilers/inductor_kernel_benchmark_config.ts b/torchci/components/benchmark_v3/configs/teams/compilers/inductor_kernel_benchmark_config.ts
index 395aa7f1e9..f0e257daa2 100644
--- a/torchci/components/benchmark_v3/configs/teams/compilers/inductor_kernel_benchmark_config.ts
+++ b/torchci/components/benchmark_v3/configs/teams/compilers/inductor_kernel_benchmark_config.ts
@@ -1,48 +1,17 @@
-import {
- BenchmarkUIConfig,
- SubSectionRenderConfig,
- UIRenderConfig,
-} from "../../config_book_types";
-import { BenchmarkComparisonPolicyConfig } from "../../helpers/RegressionPolicy";
+import { BenchmarkUIConfig } from "../../config_book_types";
import {
DEFAULT_DASHBOARD_BENCHMARK_INITIAL,
defaultDashboardBenchmarkUIConfig,
} from "../defaults/default_dashboard_config";
export const BETTER_BENCHMARK_ID = "better_benchmark";
-
-const LOWER_IS_BETTER_POLICY: BenchmarkComparisonPolicyConfig = {
- target: "latency_us",
- type: "ratio",
- ratioPolicy: {
- badRatio: 1.05,
- goodRatio: 0.95,
- direction: "down",
- },
-};
-
-const COMPARISON_POLICY = {
- latency_us: LOWER_IS_BETTER_POLICY,
- gap_vs_sol: {
- ...LOWER_IS_BETTER_POLICY,
- target: "gap_vs_sol",
- },
-};
-
-function withComparisonPolicy(render: UIRenderConfig): UIRenderConfig {
- if (render.type !== "AutoBenchmarkTimeSeriesTable") {
- return render;
- }
- return {
- ...render,
- config: {
- ...render.config,
- comparisonPolicy: COMPARISON_POLICY,
- },
- };
-}
+export const BETTER_BENCHMARK_SUMMARY_FETCHER_ID = "better_benchmark_summary";
const defaultDataRender = defaultDashboardBenchmarkUIConfig.dataRender;
+const externalLinkRenders = (defaultDataRender.renders ?? []).filter(
+ (render: { type: string }) =>
+ render.type === "AutoBenchmarkComparisonGithubExternalLink"
+);
export const BetterBenchmarkDashboardConfig: BenchmarkUIConfig = {
...defaultDashboardBenchmarkUIConfig,
@@ -64,20 +33,28 @@ export const BetterBenchmarkDashboardConfig: BenchmarkUIConfig = {
},
dataRender: {
...defaultDataRender,
- renders: (defaultDataRender.renders ?? []).map(withComparisonPolicy),
- subSectionRenders: Object.fromEntries(
- (
- Object.entries(defaultDataRender.subSectionRenders ?? {}) as [
- string,
- SubSectionRenderConfig
- ][]
- ).map(([name, section]) => [
- name,
- {
- ...section,
- renders: section.renders.map(withComparisonPolicy),
+ renders: [
+ {
+ type: "AutoBetterBenchmarkSummary",
+ title: "Full performance rollup",
+ config: {
+ fetcherId: BETTER_BENCHMARK_SUMMARY_FETCHER_ID,
+ },
+ },
+ ...externalLinkRenders,
+ ],
+ subSectionRenders: {
+ main: {
+ filterConstraint: {
+ mode: {
+ disableOptions: ["inference"],
+ },
+ dtype: {
+ disableOptions: ["unknown"],
+ },
},
- ])
- ),
+ renders: [],
+ },
+ },
},
};
diff --git a/torchci/components/benchmark_v3/configs/utils/autoRegistration.tsx b/torchci/components/benchmark_v3/configs/utils/autoRegistration.tsx
index 488470131a..8a0e281d91 100644
--- a/torchci/components/benchmark_v3/configs/utils/autoRegistration.tsx
+++ b/torchci/components/benchmark_v3/configs/utils/autoRegistration.tsx
@@ -10,6 +10,7 @@ import {
AutoBenchmarkTimeSeriesChartGroup,
AutoBenchmarkTimeSeriesTable,
} from "components/benchmark_v3/components/dataRender/auto/autoComponents";
+import { AutoBetterBenchmarkSummary } from "components/benchmark_v3/configs/teams/compilers/BetterBenchmarkSummary";
export type AutoComponentProps = {
config?: any;
@@ -48,6 +49,9 @@ export class AutoComponentRegistry {
private constructor() {
const registry: Record = {
+ AutoBetterBenchmarkSummary: {
+ Component: AutoBetterBenchmarkSummary,
+ },
AutoBenchmarkTimeSeriesTable: {
Component: AutoBenchmarkTimeSeriesTable,
},
diff --git a/torchci/lib/benchmark/api_helper/backend/dataFetchers/fetchers.ts b/torchci/lib/benchmark/api_helper/backend/dataFetchers/fetchers.ts
index 5050ef76b8..028645fb22 100644
--- a/torchci/lib/benchmark/api_helper/backend/dataFetchers/fetchers.ts
+++ b/torchci/lib/benchmark/api_helper/backend/dataFetchers/fetchers.ts
@@ -1,5 +1,6 @@
import {
BenchmarkDataQuery,
+ BetterBenchmarkDataFetcher,
PytorchAoMicroApiBenchmarkDataFetcher,
PytorchHelionDataFetcher,
PytorchOperatorMicroBenchmarkDataFetcher,
@@ -29,6 +30,7 @@ import {
// Register benchmark data fetchers, this is mainly used in get_benchmark_data api and get_time_series api
const dataCtors: Record BenchmarkDataFetcher> = {
+ better_benchmark_summary: BetterBenchmarkDataFetcher,
pytorch_operator_microbenchmark: PytorchOperatorMicroBenchmarkDataFetcher,
pytorch_helion: PytorchHelionDataFetcher,
torchao_micro_api_benchmark: PytorchAoMicroApiBenchmarkDataFetcher,
diff --git a/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/benchmarkDataQueryBuilder.ts b/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/benchmarkDataQueryBuilder.ts
index 4da831449e..ea1a72f5ea 100644
--- a/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/benchmarkDataQueryBuilder.ts
+++ b/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/benchmarkDataQueryBuilder.ts
@@ -1,6 +1,10 @@
import { deepClone } from "@mui/x-data-grid/internals";
import { toBenchmarkTimeSeriesReponseFormat } from "../../common/utils";
import { BenchmarkDataFetcher } from "../type";
+import {
+ buildBetterBenchmarkSummary,
+ RawBenchmarkRow,
+} from "./betterBenchmarkSummary";
import { ExecutableQueryBase, QueryBuilder, SelectItem } from "./queryBuilder";
const DEFAULT_TS_GROUP_KEY = [
@@ -115,6 +119,7 @@ export class BenchmarkDataQuery extends ExecutableQueryBase {
SELECT
replaceOne(o.head_branch, 'refs/heads/', '') AS branch,
o.workflow_id AS workflow_id,
+ o.run_attempt AS run_attempt,
o.job_id AS job_id,
o.repo AS repo,
o.head_sha AS commit,
@@ -193,6 +198,7 @@ export class BenchmarkDataQuery extends ExecutableQueryBase {
`
SELECT DISTINCT
workflow_id,
+ run_attempt,
repo,
branch,
commit,
@@ -380,6 +386,96 @@ export class BenchmarkDataQuery extends ExecutableQueryBase {
}
}
+export class BetterBenchmarkDataFetcher extends BenchmarkDataQuery {
+ private workflowIds: Array = [];
+
+ constructor() {
+ super();
+ this.replaceValueSelectStatement(
+ "toFloat64(arrayAvg(o.metric.'benchmark_values'))"
+ );
+ this.addExtraInfos(
+ new Map(
+ [
+ "record_type",
+ "pattern_hash",
+ "shape_hash",
+ "kernel_name",
+ "suite",
+ "source_mode",
+ "example_model",
+ "included",
+ "exclusion_reasons",
+ "timing_policy",
+ "accounting_digest",
+ "model_accounting_digest",
+ "sweep_total_repros",
+ "sweep_failed_repros",
+ "sweep_invalid_measurements",
+ "sweep_missing_shape_files",
+ "sweep_unresolved_shape_metadata",
+ ].map((key) => [
+ key,
+ `tupleElement(o.benchmark, 'extra_info')['${key}']`,
+ ])
+ )
+ );
+ }
+
+ toQueryParams(inputs: any, id?: string): Record {
+ const params = super.toQueryParams(inputs, id);
+ this.workflowIds = params.workflows ?? [];
+ return params;
+ }
+
+ applyFormat(
+ data: RawBenchmarkRow[],
+ formats: string[],
+ includesAllExtraKey: boolean = true,
+ groupByFields?: string[]
+ ): any {
+ const kernelRows = data.filter(
+ (row) =>
+ !row.extra_key?.record_type || row.extra_key.record_type === "kernel"
+ );
+ const standardFormats = formats.filter(
+ (format) => format !== "better_summary"
+ );
+ const timestamps = data
+ .map((row) => Date.parse(row.metadata_info?.timestamp ?? ""))
+ .filter(Number.isFinite);
+ const start = timestamps.length
+ ? new Date(Math.min(...timestamps)).toISOString()
+ : null;
+ const end = timestamps.length
+ ? new Date(Math.max(...timestamps)).toISOString()
+ : null;
+ const standard: any =
+ standardFormats.length > 0
+ ? super.applyFormat(
+ kernelRows,
+ standardFormats,
+ includesAllExtraKey,
+ groupByFields
+ )
+ : {
+ total_raw_rows: data.length,
+ time_range: {
+ start,
+ end,
+ },
+ data: {},
+ };
+ if (formats.includes("better_summary")) {
+ standard.data.better_summary = buildBetterBenchmarkSummary(
+ data,
+ this.workflowIds
+ );
+ }
+ return standard;
+ }
+}
+
/**
* helper function to convert a map of statements to a query map result
* e.g. map('key1', value1, 'key2', value2, ...)
diff --git a/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/betterBenchmarkDataFetcher.test.ts b/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/betterBenchmarkDataFetcher.test.ts
new file mode 100644
index 0000000000..44333ee8b0
--- /dev/null
+++ b/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/betterBenchmarkDataFetcher.test.ts
@@ -0,0 +1,452 @@
+/** @jest-environment node */
+
+import { BetterBenchmarkDashboardConfig } from "components/benchmark_v3/configs/teams/compilers/inductor_kernel_benchmark_config";
+
+import { getBenchmarkDataFetcher } from "../fetchers";
+import { buildBetterBenchmarkSummary } from "./betterBenchmarkSummary";
+
+function row(
+ workflow_id: number,
+ model: string,
+ metric: string,
+ value: number,
+ record_type: "kernel" | "model",
+ extra: Record = {}
+) {
+ return {
+ workflow_id,
+ model,
+ metric,
+ value,
+ extra_key: { record_type, ...extra },
+ };
+}
+
+describe("buildBetterBenchmarkSummary", () => {
+ test("computes shape-aligned kernel and model reductions", () => {
+ const data = [
+ row(10, "kernel-a", "latency_us", 10, "kernel"),
+ row(20, "kernel-a", "latency_us", 5, "kernel"),
+ row(10, "kernel-a", "gap_vs_sol", 4, "kernel"),
+ row(20, "kernel-a", "gap_vs_sol", 2, "kernel"),
+ row(10, "kernel-base-only", "latency_us", 4, "kernel"),
+ row(
+ 10,
+ "timm/infer/model-a",
+ "projected_model_latency_us",
+ 100,
+ "model",
+ {
+ suite: "timm",
+ source_mode: "infer",
+ }
+ ),
+ row(20, "timm/infer/model-a", "projected_model_latency_us", 80, "model", {
+ suite: "timm",
+ source_mode: "infer",
+ }),
+ row(10, "timm/infer/model-a", "model_coverage_ratio", 1, "model"),
+ row(10, "hf/train/model-b", "model_coverage_ratio", 0.5, "model"),
+ ];
+
+ const summary = buildBetterBenchmarkSummary(data, [10, 20]);
+
+ expect(summary.kernel).toEqual({
+ geomean: 100,
+ median: 50,
+ mean: 50,
+ n: 1,
+ });
+ expect(summary.model).toEqual({
+ geomean: 25,
+ median: 20,
+ mean: 20,
+ n: 1,
+ });
+ expect(summary.models[0]).toEqual(
+ expect.objectContaining({ baseUs: 100, headUs: 80, deltaUs: 20 })
+ );
+ expect(summary.kernels[0]).toEqual(
+ expect.objectContaining({
+ baseUs: 10,
+ headUs: 5,
+ deltaUs: 5,
+ baseGapVsSol: 4,
+ headGapVsSol: 2,
+ })
+ );
+ expect(summary.suites).toEqual([
+ expect.objectContaining({
+ suiteMode: "timm/infer",
+ stats: expect.objectContaining({ geomean: 25, n: 1 }),
+ }),
+ expect.objectContaining({
+ suiteMode: "hf/train",
+ stats: null,
+ excluded: 1,
+ }),
+ ]);
+ expect(summary.coverage).toEqual({
+ matchedKernelPoints: 1,
+ leftOnlyKernelPoints: 1,
+ rightOnlyKernelPoints: 0,
+ invalidBaselineKernelPoints: 0,
+ invalidCandidateKernelPoints: 0,
+ incompatibleModels: 0,
+ includedModels: 1,
+ totalModels: 2,
+ baselineModels: {
+ total: 2,
+ included: 0,
+ excluded: 2,
+ meanCoverage: 0.75,
+ exclusionReasons: {},
+ },
+ candidateModels: {
+ total: 0,
+ included: 0,
+ excluded: 0,
+ meanCoverage: 0,
+ exclusionReasons: {},
+ },
+ baselineRun: {
+ totalRepros: null,
+ failedRepros: null,
+ invalidMeasurements: null,
+ missingShapeFiles: null,
+ unresolvedShapeMetadata: null,
+ available: false,
+ },
+ candidateRun: {
+ totalRepros: null,
+ failedRepros: null,
+ invalidMeasurements: null,
+ missingShapeFiles: null,
+ unresolvedShapeMetadata: null,
+ available: false,
+ },
+ });
+ });
+
+ test("compares a workflow with itself", () => {
+ const data = [row(10, "kernel-a", "latency_us", 10, "kernel")];
+
+ const summary = buildBetterBenchmarkSummary(data, [10, 10]);
+
+ expect(summary.kernel).toEqual({
+ geomean: 0,
+ median: 0,
+ mean: 0,
+ n: 1,
+ });
+ });
+
+ test("normalizes legacy and new kernel identities", () => {
+ const data = [
+ {
+ workflow_id: 10,
+ model: "pointwise_abcdef123456[resnet18_1234abcd]",
+ metric: "latency_us",
+ value: 10,
+ extra_key: {
+ timing_policy: "compiled_us",
+ } as Record,
+ },
+ {
+ workflow_id: 20,
+ model: "pointwise_abcdef123456[1234abcd]",
+ metric: "latency_us",
+ value: 8,
+ extra_key: {
+ record_type: "kernel",
+ pattern_hash: "abcdef123456",
+ shape_hash: "1234abcd",
+ timing_policy: "compiled_us",
+ } as Record,
+ },
+ ];
+
+ const summary = buildBetterBenchmarkSummary(data, [10, 20]);
+
+ expect(summary.kernel?.median).toBe(20);
+ expect(summary.kernels).toHaveLength(1);
+ });
+
+ test("rejects legacy compiled timing against new auto timing", () => {
+ const data = [
+ {
+ workflow_id: 10,
+ model: "pointwise_abcdef123456[resnet18_1234abcd]",
+ metric: "latency_us",
+ value: 10,
+ extra_key: {},
+ },
+ row(20, "pointwise_abcdef123456[1234abcd]", "latency_us", 8, "kernel", {
+ pattern_hash: "abcdef123456",
+ shape_hash: "1234abcd",
+ timing_policy: "auto",
+ }),
+ ];
+
+ const summary = buildBetterBenchmarkSummary(data, [10, 20]);
+ expect(summary.kernel).toBeNull();
+ expect(summary.comparisonUnavailableReason).toContain(
+ "different timing policy"
+ );
+ });
+
+ test("excludes genai microbenchmarks only from the headline", () => {
+ const data = [
+ row(10, "timm/infer/model-a", "projected_model_latency_us", 100, "model"),
+ row(20, "timm/infer/model-a", "projected_model_latency_us", 80, "model"),
+ row(
+ 10,
+ "genai/static/SoftmaxForward",
+ "projected_model_latency_us",
+ 100,
+ "model"
+ ),
+ row(
+ 20,
+ "genai/static/SoftmaxForward",
+ "projected_model_latency_us",
+ 50,
+ "model"
+ ),
+ ];
+
+ const summary = buildBetterBenchmarkSummary(data, [10, 20]);
+
+ expect(summary.model?.n).toBe(1);
+ expect(summary.model?.median).toBe(20);
+ expect(summary.models).toHaveLength(1);
+ expect(summary.suites.map((suite) => suite.suiteMode)).not.toContain(
+ "genai/static"
+ );
+ });
+
+ test("rejects an implicit workflow comparison", () => {
+ const data = [row(10, "kernel-a", "latency_us", 10, "kernel")];
+
+ expect(() => buildBetterBenchmarkSummary(data, [])).toThrow(
+ "requires explicit left/right workflows"
+ );
+ });
+
+ test("retains every shard job in a workflow", () => {
+ const data = [
+ {
+ ...row(10, "kernel-a", "latency_us", 10, "kernel"),
+ job_id: "base-shard-1",
+ },
+ {
+ ...row(10, "kernel-b", "latency_us", 20, "kernel"),
+ job_id: "base-shard-2",
+ },
+ {
+ ...row(20, "kernel-a", "latency_us", 5, "kernel"),
+ job_id: "head-shard-1",
+ },
+ {
+ ...row(20, "kernel-b", "latency_us", 10, "kernel"),
+ job_id: "head-shard-2",
+ },
+ ];
+
+ const summary = buildBetterBenchmarkSummary(data, [10, 20]);
+
+ expect(summary.kernel?.n).toBe(2);
+ expect(summary.kernel?.median).toBe(50);
+ });
+
+ test("uses only the latest run attempt for each workflow", () => {
+ const data = [
+ { ...row(10, "kernel-a", "latency_us", 10, "kernel"), run_attempt: 1 },
+ { ...row(10, "kernel-a", "latency_us", 8, "kernel"), run_attempt: 2 },
+ { ...row(20, "kernel-a", "latency_us", 4, "kernel"), run_attempt: 1 },
+ ];
+
+ const summary = buildBetterBenchmarkSummary(data, [10, 20]);
+
+ expect(summary.kernel?.median).toBe(50);
+ });
+
+ test("rejects incompatible accounting artifacts", () => {
+ const data = [
+ row(10, "timm/infer/model-a", "projected_model_latency_us", 10, "model", {
+ accounting_digest: "left",
+ }),
+ row(20, "timm/infer/model-a", "projected_model_latency_us", 5, "model", {
+ accounting_digest: "right",
+ }),
+ row(10, "kernel-a", "latency_us", 10, "kernel"),
+ row(20, "kernel-a", "latency_us", 5, "kernel"),
+ ];
+
+ const summary = buildBetterBenchmarkSummary(data, [10, 20]);
+ expect(summary.model).toBeNull();
+ expect(summary.kernel?.n).toBe(1);
+ expect(summary.modelUnavailableReason).toBe("");
+ expect(summary.coverage.incompatibleModels).toBe(1);
+ });
+
+ test("allows exact duplicates but rejects conflicting values", () => {
+ const exact = [
+ row(10, "kernel-a", "latency_us", 10, "kernel"),
+ row(10, "kernel-a", "latency_us", 10, "kernel"),
+ row(20, "kernel-a", "latency_us", 5, "kernel"),
+ ];
+ expect(buildBetterBenchmarkSummary(exact, [10, 20]).kernel?.n).toBe(1);
+
+ const conflicting = [
+ ...exact,
+ row(10, "kernel-a", "latency_us", 11, "kernel"),
+ ];
+ expect(() => buildBetterBenchmarkSummary(conflicting, [10, 20])).toThrow(
+ "Conflicting kernel values"
+ );
+ });
+
+ test("returns null aggregates when no exact matches exist", () => {
+ const data = [
+ row(10, "kernel-a", "latency_us", 10, "kernel"),
+ row(20, "kernel-b", "latency_us", 10, "kernel"),
+ ];
+
+ const summary = buildBetterBenchmarkSummary(data, [10, 20]);
+
+ expect(summary.kernel).toBeNull();
+ expect(summary.model).toBeNull();
+ expect(summary.kernels).toEqual([]);
+ });
+
+ test("returns an empty summary when selected workflows have no rows", () => {
+ const summary = buildBetterBenchmarkSummary([], [10, 20]);
+
+ expect(summary.kernel).toBeNull();
+ expect(summary.model).toBeNull();
+ expect(summary.coverage).toEqual({
+ matchedKernelPoints: 0,
+ leftOnlyKernelPoints: 0,
+ rightOnlyKernelPoints: 0,
+ invalidBaselineKernelPoints: 0,
+ invalidCandidateKernelPoints: 0,
+ incompatibleModels: 0,
+ includedModels: 0,
+ totalModels: 0,
+ baselineModels: {
+ total: 0,
+ included: 0,
+ excluded: 0,
+ meanCoverage: 0,
+ exclusionReasons: {},
+ },
+ candidateModels: {
+ total: 0,
+ included: 0,
+ excluded: 0,
+ meanCoverage: 0,
+ exclusionReasons: {},
+ },
+ baselineRun: {
+ totalRepros: null,
+ failedRepros: null,
+ invalidMeasurements: null,
+ missingShapeFiles: null,
+ unresolvedShapeMetadata: null,
+ available: false,
+ },
+ candidateRun: {
+ totalRepros: null,
+ failedRepros: null,
+ invalidMeasurements: null,
+ missingShapeFiles: null,
+ unresolvedShapeMetadata: null,
+ available: false,
+ },
+ });
+ });
+
+ test("surfaces invalid kernels and model exclusion details", () => {
+ const data = [
+ row(10, "kernel-invalid", "latency_us", 0, "kernel"),
+ row(20, "kernel-invalid", "latency_us", Number.NaN, "kernel"),
+ row(10, "timm/infer/good", "model_coverage_ratio", 1, "model", {
+ included: "true",
+ }),
+ row(10, "timm/infer/bad", "model_coverage_ratio", 0.5, "model", {
+ included: "false",
+ exclusion_reasons: "unmatched_kernel,trace_errors",
+ }),
+ ];
+
+ const summary = buildBetterBenchmarkSummary(data, [10, 20]);
+
+ expect(summary.coverage.invalidBaselineKernelPoints).toBe(1);
+ expect(summary.coverage.invalidCandidateKernelPoints).toBe(1);
+ expect(summary.coverage.baselineModels).toEqual({
+ total: 2,
+ included: 1,
+ excluded: 1,
+ meanCoverage: 0.75,
+ exclusionReasons: { unmatched_kernel: 1, trace_errors: 1 },
+ });
+ });
+});
+
+describe("BetterBenchmarkDataFetcher", () => {
+ test("preserves full timing precision in ClickHouse", () => {
+ const fetcher = getBenchmarkDataFetcher("better_benchmark_summary") as any;
+
+ expect(fetcher.build()).toContain(
+ "toFloat64(arrayAvg(o.metric.'benchmark_values')) AS value"
+ );
+ expect(fetcher.build()).not.toContain(
+ "floor(arrayAvg(o.metric.'benchmark_values'), 2)"
+ );
+ });
+
+ test("returns the standard response envelope", () => {
+ const fetcher = getBenchmarkDataFetcher("better_benchmark_summary") as any;
+ fetcher.toQueryParams({
+ repo: "pytorch/pytorch",
+ benchmarkName: "inductor-kernel-benchmark",
+ workflows: [10, 20],
+ });
+ const data = [
+ row(10, "kernel-a", "latency_us", 10, "kernel"),
+ row(20, "kernel-a", "latency_us", 5, "kernel"),
+ ];
+
+ const response = fetcher.applyFormat(data, ["better_summary"]);
+
+ expect(response.total_raw_rows).toBe(2);
+ expect(response.time_range.start).toBeNull();
+ expect(response.data.better_summary.kernel.n).toBe(1);
+ });
+});
+
+describe("BetterBenchmarkDashboardConfig", () => {
+ test("uses type-aware summary tables instead of the mixed generic table", () => {
+ const renderTypes =
+ BetterBenchmarkDashboardConfig.dataRender.renders?.map(
+ (render) => render.type
+ ) ?? [];
+
+ expect(renderTypes).toEqual([
+ "AutoBetterBenchmarkSummary",
+ "AutoBenchmarkComparisonGithubExternalLink",
+ ]);
+ expect(BetterBenchmarkDashboardConfig.dataRender.subSectionRenders).toEqual(
+ {
+ main: {
+ filterConstraint: {
+ mode: { disableOptions: ["inference"] },
+ dtype: { disableOptions: ["unknown"] },
+ },
+ renders: [],
+ },
+ }
+ );
+ });
+});
diff --git a/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/betterBenchmarkSummary.ts b/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/betterBenchmarkSummary.ts
new file mode 100644
index 0000000000..a2d29e56b7
--- /dev/null
+++ b/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/betterBenchmarkSummary.ts
@@ -0,0 +1,511 @@
+export type RawBenchmarkRow = {
+ workflow_id: string | number;
+ run_attempt?: string | number;
+ job_id?: string | number;
+ model: string;
+ metric: string;
+ value: number;
+ device?: string;
+ arch?: string;
+ extra_key?: Record;
+ metadata_info?: Record;
+};
+
+type Pair = {
+ id: string;
+ name: string;
+ baseUs: number;
+ headUs: number;
+ reductionPct: number;
+ speedup: number;
+ suite: string;
+ mode: string;
+ extra: Record;
+};
+
+export type RollupStats = {
+ geomean: number;
+ median: number;
+ mean: number;
+ n: number;
+};
+
+function median(values: number[]): number {
+ const sorted = [...values].sort((a, b) => a - b);
+ const middle = Math.floor(sorted.length / 2);
+ return sorted.length % 2
+ ? sorted[middle]
+ : (sorted[middle - 1] + sorted[middle]) / 2;
+}
+
+function round(value: number): number {
+ return Math.round(value * 10000) / 10000;
+}
+
+function summarize(pairs: Pair[]): RollupStats | null {
+ if (pairs.length === 0) {
+ return null;
+ }
+ const reductions = pairs.map((pair) => pair.reductionPct);
+ const logSpeedups = pairs.map((pair) => Math.log(pair.speedup));
+ return {
+ geomean: round(
+ (Math.exp(
+ logSpeedups.reduce((sum, value) => sum + value, 0) / pairs.length
+ ) -
+ 1) *
+ 100
+ ),
+ median: round(median(reductions)),
+ mean: round(
+ reductions.reduce((sum, value) => sum + value, 0) / reductions.length
+ ),
+ n: pairs.length,
+ };
+}
+
+function isGenai(name: string): boolean {
+ return name.split("/", 1)[0].toLowerCase() === "genai";
+}
+
+function kernelIdentity(row: RawBenchmarkRow): string {
+ const patternHash = row.extra_key?.pattern_hash;
+ const shapeHash = row.extra_key?.shape_hash;
+ if (patternHash && shapeHash) {
+ return `${patternHash}/${shapeHash}`;
+ }
+
+ // Legacy records used repro_dir[model_shapehash], whereas new records use
+ // repro_dir[shapehash]. Normalize both to repro_dir[shapehash].
+ const match = row.model.match(/^(.*)\[([^\]]+)\]$/);
+ if (!match) {
+ return row.model;
+ }
+ const shape = match[2].split("_").at(-1) ?? match[2];
+ const pattern = match[1].split("_").at(-1) ?? "";
+ if (/^[0-9a-f]{12}$/i.test(pattern)) {
+ return `${pattern}/${shape}`;
+ }
+ return `${match[1]}[${shape}]`;
+}
+
+function selectWorkflowRows(
+ rows: RawBenchmarkRow[],
+ workflows: string[]
+): RawBenchmarkRow[] {
+ const selected = new Set(workflows);
+ const latestAttempts = new Map();
+ for (const row of rows) {
+ const workflow = String(row.workflow_id);
+ if (!selected.has(workflow)) {
+ continue;
+ }
+ const attempt = Number(row.run_attempt ?? 1);
+ latestAttempts.set(
+ workflow,
+ Math.max(latestAttempts.get(workflow) ?? 1, attempt)
+ );
+ }
+ return rows.filter(
+ (row) =>
+ selected.has(String(row.workflow_id)) &&
+ Number(row.run_attempt ?? 1) ===
+ latestAttempts.get(String(row.workflow_id))
+ );
+}
+
+function singleMetadata(
+ rows: RawBenchmarkRow[],
+ workflow: string,
+ read: (_row: RawBenchmarkRow) => string | undefined,
+ label: string
+): string {
+ const values = new Set(
+ rows
+ .filter((row) => String(row.workflow_id) === workflow)
+ .map(read)
+ .filter((value): value is string => Boolean(value))
+ );
+ if (values.size > 1) {
+ throw new Error(
+ `Workflow ${workflow} has conflicting ${label}: ${[...values].join(", ")}`
+ );
+ }
+ return [...values][0] ?? "";
+}
+
+function validateCompatibility(
+ rows: RawBenchmarkRow[],
+ leftWorkflow: string,
+ rightWorkflow: string
+): string {
+ for (const [read, label, strict] of [
+ [(row: RawBenchmarkRow) => row.device, "device", false],
+ [(row: RawBenchmarkRow) => row.arch, "architecture", false],
+ [
+ (row: RawBenchmarkRow) => row.extra_key?.timing_policy,
+ "timing policy",
+ true,
+ ],
+ ] as const) {
+ const left = singleMetadata(rows, leftWorkflow, read, label);
+ const right = singleMetadata(rows, rightWorkflow, read, label);
+ if (
+ (strict && left !== right) ||
+ (!strict && left && right && left !== right)
+ ) {
+ return `Cannot compare workflows with different ${label}: ${left} vs ${right}`;
+ }
+ }
+ return "";
+}
+
+function indexedRows(
+ rows: RawBenchmarkRow[],
+ workflow: string,
+ recordType: "kernel" | "model",
+ metric: string
+): Map {
+ const points = new Map();
+ for (const row of rows) {
+ const rowRecordType = row.extra_key?.record_type || "kernel";
+ if (
+ String(row.workflow_id) !== workflow ||
+ row.metric !== metric ||
+ rowRecordType !== recordType ||
+ !Number.isFinite(Number(row.value)) ||
+ Number(row.value) <= 0
+ ) {
+ continue;
+ }
+ const id = recordType === "kernel" ? kernelIdentity(row) : row.model;
+ const previous = points.get(id);
+ if (previous && Number(previous.value) !== Number(row.value)) {
+ throw new Error(
+ `Conflicting ${recordType} values for workflow ${workflow}: ${id}`
+ );
+ }
+ points.set(id, row);
+ }
+ return points;
+}
+
+function invalidRowCount(
+ rows: RawBenchmarkRow[],
+ workflow: string,
+ recordType: "kernel" | "model",
+ metric: string
+): number {
+ return rows.filter((row) => {
+ const rowRecordType = row.extra_key?.record_type || "kernel";
+ return (
+ String(row.workflow_id) === workflow &&
+ row.metric === metric &&
+ rowRecordType === recordType &&
+ (!Number.isFinite(Number(row.value)) || Number(row.value) <= 0)
+ );
+ }).length;
+}
+
+function pairedRows(
+ rows: RawBenchmarkRow[],
+ leftWorkflow: string,
+ rightWorkflow: string,
+ recordType: "kernel" | "model",
+ metric: string
+): {
+ pairs: Pair[];
+ leftOnly: number;
+ rightOnly: number;
+ leftInvalid: number;
+ rightInvalid: number;
+ incompatible: number;
+} {
+ const base = indexedRows(rows, leftWorkflow, recordType, metric);
+ const head = indexedRows(rows, rightWorkflow, recordType, metric);
+ const pairs: Pair[] = [];
+ let incompatible = 0;
+ for (const [id, baseRow] of base.entries()) {
+ const headRow = head.get(id);
+ if (!headRow) {
+ continue;
+ }
+ if (recordType === "model") {
+ const baseAccounting =
+ baseRow.extra_key?.model_accounting_digest ||
+ baseRow.extra_key?.accounting_digest ||
+ "";
+ const headAccounting =
+ headRow.extra_key?.model_accounting_digest ||
+ headRow.extra_key?.accounting_digest ||
+ "";
+ if (
+ baseAccounting &&
+ headAccounting &&
+ baseAccounting !== headAccounting
+ ) {
+ incompatible += 1;
+ continue;
+ }
+ }
+ const baseUs = Number(baseRow.value);
+ const headUs = Number(headRow.value);
+ const reductionPct = (1 - headUs / baseUs) * 100;
+ const nameParts = baseRow.model.split("/", 3);
+ const suite =
+ recordType === "model"
+ ? nameParts[0] || "unknown"
+ : baseRow.extra_key?.suite || "unknown";
+ const mode =
+ recordType === "model"
+ ? nameParts[1] || "unknown"
+ : baseRow.extra_key?.source_mode || "unknown";
+ pairs.push({
+ id,
+ name: baseRow.model,
+ baseUs,
+ headUs,
+ reductionPct,
+ speedup: baseUs / headUs,
+ suite,
+ mode,
+ extra: baseRow.extra_key ?? {},
+ });
+ }
+ return {
+ pairs,
+ leftOnly: [...base.keys()].filter((id) => !head.has(id)).length,
+ rightOnly: [...head.keys()].filter((id) => !base.has(id)).length,
+ leftInvalid: invalidRowCount(rows, leftWorkflow, recordType, metric),
+ rightInvalid: invalidRowCount(rows, rightWorkflow, recordType, metric),
+ incompatible,
+ };
+}
+
+function modelCoverageSummary(rows: RawBenchmarkRow[], workflow: string) {
+ const coverageRows = rows.filter(
+ (row) =>
+ row.metric === "model_coverage_ratio" &&
+ row.extra_key?.record_type === "model" &&
+ String(row.workflow_id) === workflow &&
+ !isGenai(row.model)
+ );
+ const reasons: Record = {};
+ let included = 0;
+ let coverageTotal = 0;
+ for (const row of coverageRows) {
+ const value = Number(row.value);
+ if (Number.isFinite(value)) {
+ coverageTotal += value;
+ }
+ if (row.extra_key?.included === "true") {
+ included += 1;
+ }
+ for (const reason of (row.extra_key?.exclusion_reasons ?? "")
+ .split(",")
+ .filter(Boolean)) {
+ reasons[reason] = (reasons[reason] ?? 0) + 1;
+ }
+ }
+ return {
+ total: coverageRows.length,
+ included,
+ excluded: coverageRows.length - included,
+ meanCoverage:
+ coverageRows.length > 0 ? round(coverageTotal / coverageRows.length) : 0,
+ exclusionReasons: reasons,
+ };
+}
+
+function runQualitySummary(rows: RawBenchmarkRow[], workflow: string) {
+ const readNumber = (key: string) => {
+ const raw = singleMetadata(
+ rows,
+ workflow,
+ (row) => row.extra_key?.[key],
+ key
+ );
+ if (!raw) {
+ return null;
+ }
+ const value = Number(raw);
+ return Number.isFinite(value) && value >= 0 ? value : null;
+ };
+ const summary = {
+ totalRepros: readNumber("sweep_total_repros"),
+ failedRepros: readNumber("sweep_failed_repros"),
+ invalidMeasurements: readNumber("sweep_invalid_measurements"),
+ missingShapeFiles: readNumber("sweep_missing_shape_files"),
+ unresolvedShapeMetadata: readNumber("sweep_unresolved_shape_metadata"),
+ };
+ return {
+ ...summary,
+ available: Object.values(summary).some((value) => value !== null),
+ };
+}
+
+export function buildBetterBenchmarkSummary(
+ rawRows: RawBenchmarkRow[],
+ workflowIds: Array
+) {
+ if (workflowIds.length !== 2) {
+ throw new Error(
+ "Better Benchmark summary requires explicit left/right workflows"
+ );
+ }
+ const [leftWorkflow, rightWorkflow] = workflowIds.map(String);
+ for (const workflow of [leftWorkflow, rightWorkflow]) {
+ if (!/^\d+$/.test(workflow) || Number(workflow) <= 0) {
+ throw new Error(`Invalid workflow id: ${workflow}`);
+ }
+ }
+
+ const rows = selectWorkflowRows(rawRows, [leftWorkflow, rightWorkflow]);
+ const comparisonUnavailableReason = validateCompatibility(
+ rows,
+ leftWorkflow,
+ rightWorkflow
+ );
+ const emptyResult = {
+ pairs: [],
+ leftOnly: 0,
+ rightOnly: 0,
+ leftInvalid: 0,
+ rightInvalid: 0,
+ incompatible: 0,
+ };
+ const modelUnavailableReason = comparisonUnavailableReason;
+
+ const modelResult = modelUnavailableReason
+ ? emptyResult
+ : pairedRows(
+ rows,
+ leftWorkflow,
+ rightWorkflow,
+ "model",
+ "projected_model_latency_us"
+ );
+ const kernelResult = comparisonUnavailableReason
+ ? emptyResult
+ : pairedRows(rows, leftWorkflow, rightWorkflow, "kernel", "latency_us");
+ const models = modelResult.pairs;
+ const realModels = models.filter((model) => !isGenai(model.name));
+ const kernels = kernelResult.pairs;
+ const baselineKernelGaps = indexedRows(
+ rows,
+ leftWorkflow,
+ "kernel",
+ "gap_vs_sol"
+ );
+ const candidateKernelGaps = indexedRows(
+ rows,
+ rightWorkflow,
+ "kernel",
+ "gap_vs_sol"
+ );
+
+ const suiteGroups = new Map();
+ for (const model of realModels) {
+ const key = `${model.suite}/${model.mode}`;
+ const group = suiteGroups.get(key) ?? [];
+ group.push(model);
+ suiteGroups.set(key, group);
+ }
+
+ const coverageRows = rows.filter(
+ (row) =>
+ row.metric === "model_coverage_ratio" &&
+ row.extra_key?.record_type === "model"
+ );
+ const realCoverageRows = coverageRows.filter((row) => !isGenai(row.model));
+ const coverageModels = new Set(realCoverageRows.map((row) => row.model));
+ const coverageBySuite = new Map>();
+ for (const row of realCoverageRows) {
+ const nameParts = row.model.split("/", 3);
+ const suite = row.extra_key?.suite || nameParts[0] || "unknown";
+ const mode = row.extra_key?.source_mode || nameParts[1] || "unknown";
+ const key = `${suite}/${mode}`;
+ const group = coverageBySuite.get(key) ?? new Set();
+ group.add(row.model);
+ coverageBySuite.set(key, group);
+ }
+
+ const toMover = (
+ pair: Pair,
+ gaps: { baseline: number | null; candidate: number | null } = {
+ baseline: null,
+ candidate: null,
+ }
+ ) => ({
+ id: pair.id,
+ name: pair.name,
+ suite: pair.suite,
+ mode: pair.mode,
+ baseUs: round(pair.baseUs),
+ headUs: round(pair.headUs),
+ deltaUs: round(pair.baseUs - pair.headUs),
+ reductionPct: round(pair.reductionPct),
+ speedup: round(pair.speedup),
+ patternHash: pair.extra.pattern_hash ?? "",
+ shapeHash: pair.extra.shape_hash ?? "",
+ exampleModel: pair.extra.example_model ?? "",
+ baseGapVsSol: gaps.baseline == null ? null : round(gaps.baseline),
+ headGapVsSol: gaps.candidate == null ? null : round(gaps.candidate),
+ });
+
+ return {
+ comparison: {
+ leftWorkflow,
+ rightWorkflow,
+ },
+ comparisonUnavailableReason,
+ modelUnavailableReason,
+ model: summarize(realModels),
+ kernel: summarize(kernels),
+ suites: [...new Set([...suiteGroups.keys(), ...coverageBySuite.keys()])]
+ .map((suiteMode) => {
+ const pairs = suiteGroups.get(suiteMode) ?? [];
+ const total = coverageBySuite.get(suiteMode)?.size ?? pairs.length;
+ return {
+ id: suiteMode,
+ suiteMode,
+ stats: summarize(pairs),
+ total,
+ excluded: Math.max(0, total - pairs.length),
+ };
+ })
+ .sort(
+ (a, b) =>
+ (b.stats?.geomean ?? -Infinity) - (a.stats?.geomean ?? -Infinity)
+ ),
+ models: realModels.map((pair) => toMover(pair)),
+ kernels: kernels.map((pair) =>
+ toMover(pair, {
+ baseline: baselineKernelGaps.has(pair.id)
+ ? Number(baselineKernelGaps.get(pair.id)?.value)
+ : null,
+ candidate: candidateKernelGaps.has(pair.id)
+ ? Number(candidateKernelGaps.get(pair.id)?.value)
+ : null,
+ })
+ ),
+ coverage: {
+ matchedKernelPoints: kernels.length,
+ leftOnlyKernelPoints: kernelResult.leftOnly,
+ rightOnlyKernelPoints: kernelResult.rightOnly,
+ invalidBaselineKernelPoints: kernelResult.leftInvalid,
+ invalidCandidateKernelPoints: kernelResult.rightInvalid,
+ incompatibleModels: modelResult.incompatible,
+ includedModels: realModels.length,
+ totalModels: coverageModels.size || realModels.length,
+ baselineModels: modelCoverageSummary(rows, leftWorkflow),
+ candidateModels: modelCoverageSummary(rows, rightWorkflow),
+ baselineRun: runQualitySummary(rows, leftWorkflow),
+ candidateRun: runQualitySummary(rows, rightWorkflow),
+ },
+ };
+}
+
+export type BetterBenchmarkSummaryData = ReturnType<
+ typeof buildBetterBenchmarkSummary
+>;