[Helion Benchmark] Use geomean for speedup

stack-info: PR: #7126, branch: oulgen/stack/6

Author: oulgen

torchci/clickhouse_queries/oss_ci_benchmark_llms/query.sql

@@ -10,7 +10,10 @@ WITH benchmarks AS (
         o.model.'backend' AS backend,
         o.model.'origins' AS origins,
         o.metric.'name' AS metric,
+        -- Arithmetic mean
         floor(arrayAvg(o.metric.'benchmark_values'), 2) AS actual,
+        -- Geometric mean
+        floor(exp(arrayAvg(arrayMap(x -> log(x), o.metric.'benchmark_values'))), 2) AS actual_geomean,
         floor(toFloat64(o.metric.'target_value'), 2) AS target,
         o.benchmark.'mode' AS mode,
         o.benchmark.'dtype' AS dtype,
@@ -141,6 +144,7 @@ SELECT DISTINCT
     origins,
     metric,
     actual,
+    actual_geomean,
     target,
     mode,
     dtype,

torchci/components/benchmark/llms/components/LLMsSummaryPanel.tsx

@@ -384,6 +384,9 @@ export default function LLMsSummaryPanel({
                   l = l === 1 ? "Pass" : "Fail";
                   r = r === 1 ? "Pass" : "Fail";
                 } else if (metric.includes("speedup")) {
+                  l = v.l.actual_geomean;
+                  r = v.r.actual_geomean;
+
                   const accuracy = metric.replace(/_speedup$/, "_accuracy");
                   const accuracy_v = params.row[accuracy];
                   if (accuracy_v.l.actual !== 1) {

torchci/lib/benchmark/llms/common.ts

@@ -42,11 +42,11 @@ export const METRIC_DISPLAY_HEADERS: { [k: string]: string } = {
   p99_ttft_ms: "p99 TTFT (ms)",
   requests_per_second: "Requests/s",
   tokens_per_second: "Tokens/s",
-  triton_speedup: "Triton Speedup",
+  triton_speedup: "Triton Speedup (Geomean)",
   triton_accuracy: "Triton Accuracy",
-  torch_compile_speedup: "Torch Compile Speedup",
+  torch_compile_speedup: "Torch Compile Speedup (Geomean)",
   torch_compile_accuracy: "Torch Compile Accuracy",
-  helion_speedup: "Helion Speedup",
+  helion_speedup: "Helion Speedup (Geomean)",
   helion_accuracy: "Helion Accuracy",
 };
 // The variable name is a bit dumb, but it tells if a higher metric value
@@ -127,6 +127,7 @@ export interface LLMsBenchmarkData {
   job_id: number;
   metric: string;
   actual: number;
+  actual_geomean: number;
   target: number;
   mode?: string;
   dtype: string;

torchci/lib/benchmark/llms/utils/llmUtils.ts

@@ -184,6 +184,7 @@ export function computeGeomean(data: LLMsBenchmarkData[], metricName: string) {
       job_id: Number(jobId),
       metric: `${metric} (geomean)`,
       actual: Number(gm),
+      actual_geomean: Number(gm),
       target: 0,
       dtype: dtype,
       device: device,
@@ -339,19 +340,23 @@ const toRowData = (
         l: hasL
           ? {
               actual: Number.MAX_SAFE_INTEGER, // indicate the failure on left side
+              actual_geomean: Number.MAX_SAFE_INTEGER, // indicate the failure on left side
               target: 0,
             }
           : {
               actual: 0,
+              actual_geomean: 0,
               target: 0,
             },
         r: hasR
           ? {
               actual: Number.MAX_SAFE_INTEGER, // indicate the failure on right side
+              actual_geomean: Number.MAX_SAFE_INTEGER, // indicate the failure on right side
               target: 0,
             }
           : {
               actual: 0,
+              actual_geomean: 0,
               target: 0,
             },
         highlight: hasL && hasR,
@@ -361,19 +366,23 @@ const toRowData = (
         l: hasL
           ? {
               actual: record["l"].actual,
+              actual_geomean: record["l"].actual_geomean,
               target: record["l"].target,
             }
           : {
               actual: 0,
+              actual_geomean: 0,
               target: 0,
             },
         r: hasR
           ? {
               actual: record["r"].actual,
+              actual_geomean: record["r"].actual_geomean,
               target: record["r"].target,
             }
           : {
               actual: 0,
+              actual_geomean: 0,
               target: 0,
             },
         highlight: hasL && hasR,