Configure PyTorch x vLLM nightly dashboard (#7614)

huydhn · web-flow · commit 8a367f515187 · 2026-01-05T18:45:52.000-08:00
A new dashboard for PyTorch x vLLM nightly benchmark after pytorch/pytorch#170466 lands ### Preview Benchmark results are showing up on https://torchci-git-pytorch-x-vllm-dashboard-fbopensource.vercel.app/benchmark/v3/dashboard/pytorch_x_vllm_benchmark?renderGroupId=main&time.start=2025-12-12T00%3A00%3A00.000Z&time.end=2025-12-19T23%3A59%3A59.999Z&lcommit.commit=7f2cac9b0c4cb1a0dc36f9c1daa7c6596c69b69a&lcommit.workflow_id=20362340534&lcommit.date=2025-12-19T23%3A00%3A00Z&lcommit.branch=setup-vllm-nightly-benchmark&rcommit.commit=7f2cac9b0c4cb1a0dc36f9c1daa7c6596c69b69a&rcommit.workflow_id=20362340534&rcommit.date=2025-12-19T23%3A00%3A00Z&rcommit.branch=setup-vllm-nightly-benchmark&lbranch=setup-vllm-nightly-benchmark&rbranch=setup-vllm-nightly-benchmark for the test run https://github.com/pytorch/pytorch/actions/runs/20362340534/job/58585533990 cc @zou3519 --------- Signed-off-by: Huy Do <huydhn@gmail.com>
diff --git a/torchci/components/benchmark_v3/configs/configurations.tsx b/torchci/components/benchmark_v3/configs/configurations.tsx
@@ -11,7 +11,7 @@ import {
 } from "components/benchmark_v3/configs/teams/helion/config";
 import {
   PYTORCH_OPERATOR_MICROBENCHMARK_ID,
-  PytorchOperatorMicroBenchmarkDashoboardConfig,
+  PytorchOperatorMicroBenchmarkDashboardConfig,
 } from "components/benchmark_v3/configs/teams/torchao/config";
 import { BenchmarkCategoryGroup } from "../components/benchmarkList/BenchmarkCategoryCard";
 import {
@@ -24,13 +24,17 @@ import {
   PytorchGptFastBenchmarkDashboardConfig,
 } from "./teams/gptfast/config";
 import {
-  PytorcAoMicroApiBenchmarkDashoboardConfig,
   PYTORCH_AO_MICRO_API_BENCHMARK_ID,
+  PytorchAoMicroApiBenchmarkDashboardConfig,
 } from "./teams/torchao/ao_micro_api_config";
 import {
-  PYTORCH_VLLM_BENCHMARK_ID,
-  PytorchVllmBenchmarkDashoboardConfig,
+  VLLM_BENCHMARK_ID,
+  VllmBenchmarkDashboardConfig,
 } from "./teams/vllm/config";
+import {
+  PYTORCH_X_VLLM_BENCHMARK_ID,
+  PytorchXVllmBenchmarkDashboardConfig,
+} from "./teams/vllm/pytorch_x_vllm_config";
 
 export const REPORT_ID_TO_BENCHMARK_ID_MAPPING: Record<string, string> = {
   compiler_regression: "compiler_inductor",
@@ -40,23 +44,26 @@ export const PREDEFINED_BENCHMARK_CONFIG: BenchmarkConfigMap = {
   [COMPILTER_BENCHMARK_NAME]: {
     [BenchmarkPageType.DashboardPage]: CompilerDashboardBenchmarkUIConfig,
   },
+  [PYTORCH_X_VLLM_BENCHMARK_ID]: {
+    [BenchmarkPageType.DashboardPage]: PytorchXVllmBenchmarkDashboardConfig,
+  },
   [COMPILTER_PRECOMPUTE_BENCHMARK_ID]: {
     [BenchmarkPageType.AggregatePage]: CompilerPrecomputeBenchmarkUIConfig,
   },
   [PYTORCH_OPERATOR_MICROBENCHMARK_ID]: {
     [BenchmarkPageType.DashboardPage]:
-      PytorchOperatorMicroBenchmarkDashoboardConfig,
+      PytorchOperatorMicroBenchmarkDashboardConfig,
   },
   [PYTORCH_HELION_BENCHMARK_ID]: {
     [BenchmarkPageType.DashboardPage]: PytorchHelionDashboardConfig,
     [BenchmarkPageType.SinglePage]: PytorchHelionSingleConfig,
   },
   [PYTORCH_AO_MICRO_API_BENCHMARK_ID]: {
     [BenchmarkPageType.DashboardPage]:
-      PytorcAoMicroApiBenchmarkDashoboardConfig,
+      PytorchAoMicroApiBenchmarkDashboardConfig,
   },
-  [PYTORCH_VLLM_BENCHMARK_ID]: {
-    [BenchmarkPageType.DashboardPage]: PytorchVllmBenchmarkDashoboardConfig,
+  [VLLM_BENCHMARK_ID]: {
+    [BenchmarkPageType.DashboardPage]: VllmBenchmarkDashboardConfig,
   },
   [PYTORCH_GPTFAST_BENCHMARK_ID]: {
     [BenchmarkPageType.DashboardPage]: PytorchGptFastBenchmarkDashboardConfig,
@@ -69,6 +76,11 @@ export const BENCHMARK_ID_MAPPING: Record<string, BenchmarkIdMappingItem> = {
     repoName: "pytorch/pytorch",
     benchmarkName: "compiler_inductor",
   },
+  [PYTORCH_X_VLLM_BENCHMARK_ID]: {
+    id: PYTORCH_X_VLLM_BENCHMARK_ID,
+    repoName: "pytorch/pytorch",
+    benchmarkName: "PyTorch x vLLM benchmark",
+  },
   [COMPILTER_PRECOMPUTE_BENCHMARK_ID]: {
     id: COMPILTER_PRECOMPUTE_BENCHMARK_ID,
     repoName: "pytorch/pytorch",
@@ -89,8 +101,8 @@ export const BENCHMARK_ID_MAPPING: Record<string, BenchmarkIdMappingItem> = {
     repoName: "pytorch/ao",
     benchmarkName: "micro-benchmark api",
   },
-  [PYTORCH_VLLM_BENCHMARK_ID]: {
-    id: PYTORCH_VLLM_BENCHMARK_ID,
+  [VLLM_BENCHMARK_ID]: {
+    id: VLLM_BENCHMARK_ID,
     repoName: "vllm-project/vllm",
     benchmarkName: "vLLM benchmark",
   },
@@ -151,6 +163,12 @@ export const BENCHMARK_CATEGORIES: BenchmarkCategoryGroup[] = [
           },
         ],
       },
+      {
+        name: "PyTorch x vLLM Benchmark",
+        route: `/benchmark/v3/dashboard/${PYTORCH_X_VLLM_BENCHMARK_ID}`,
+        info: "PyTorch x vLLM nightly benchmark using [vLLM pinned commit](https://github.com/pytorch/pytorch/blob/main/.github/ci_commit_pins/vllm.txt). Powered by [vllm-benchmark workflow](https://github.com/pytorch/pytorch/blob/main/.github/workflows/vllm-benchmark.yml) + [the benchmark configs](https://github.com/pytorch/pytorch-integration-testing/tree/main/vllm-benchmarks/benchmarks)",
+        description: "Pytorch x vLLM nightly benchmark on PyTorch",
+      },
       {
         name: "Gpt-fast Benchmark",
         route: `/benchmark/v3/dashboard/${PYTORCH_GPTFAST_BENCHMARK_ID}`,
@@ -207,13 +225,13 @@ export const BENCHMARK_CATEGORIES: BenchmarkCategoryGroup[] = [
     ],
   },
   {
-    title: "vLLM Benchmarks ",
+    title: "vLLM Benchmarks",
     tags: ["repo:vllm-project/vllm"],
     subtitle: "Benchmarks related to repo vllm-project/vllm",
     items: [
       {
         name: "VLLM V1 Benchmark",
-        route: `/benchmark/v3/dashboard/${PYTORCH_VLLM_BENCHMARK_ID}`,
+        route: `/benchmark/v3/dashboard/${VLLM_BENCHMARK_ID}`,
         info: "Powered by [code](https://github.com/pytorch/pytorch-integration-testing/tree/main/vllm-benchmarks/benchmarks)",
         actions: [
           {
diff --git a/torchci/components/benchmark_v3/configs/teams/torchao/ao_micro_api_config.ts b/torchci/components/benchmark_v3/configs/teams/torchao/ao_micro_api_config.ts
@@ -19,7 +19,7 @@ const COMPARISON_TABLE_METADATA_COLUMNS = [
   },
 ] as const;
 
-export const PytorcAoMicroApiBenchmarkDashoboardConfig: BenchmarkUIConfig = {
+export const PytorchAoMicroApiBenchmarkDashboardConfig: BenchmarkUIConfig = {
   benchmarkId: PYTORCH_AO_MICRO_API_BENCHMARK_ID,
   apiId: PYTORCH_AO_MICRO_API_BENCHMARK_ID,
   title: "TorchAo API MicroBenchmark",
diff --git a/torchci/components/benchmark_v3/configs/teams/torchao/config.ts b/torchci/components/benchmark_v3/configs/teams/torchao/config.ts
@@ -63,131 +63,130 @@ const RENDER_MAPPING_BOOK = {
   },
 };
 
-export const PytorchOperatorMicroBenchmarkDashoboardConfig: BenchmarkUIConfig =
-  {
-    benchmarkId: PYTORCH_OPERATOR_MICROBENCHMARK_ID,
-    apiId: "pytorch_operator_microbenchmark",
-    title: "Pytorch Operator MicroBenchmark Dashboard",
-    type: "dashboard",
-    dataBinding: {
-      initial: initialOptions,
-      required_filter_fields: [],
-    },
-    dataRender: {
-      type: "auto",
-      sideRender: {
-        RegressionReportFeature: {
-          type: "RegressionReportFeature",
-          title: "Regression Report Section",
-          config: {
-            report_id: "pytorch_operator_microbenchmark",
-          },
+export const PytorchOperatorMicroBenchmarkDashboardConfig: BenchmarkUIConfig = {
+  benchmarkId: PYTORCH_OPERATOR_MICROBENCHMARK_ID,
+  apiId: "pytorch_operator_microbenchmark",
+  title: "Pytorch Operator MicroBenchmark Dashboard",
+  type: "dashboard",
+  dataBinding: {
+    initial: initialOptions,
+    required_filter_fields: [],
+  },
+  dataRender: {
+    type: "auto",
+    sideRender: {
+      RegressionReportFeature: {
+        type: "RegressionReportFeature",
+        title: "Regression Report Section",
+        config: {
+          report_id: "pytorch_operator_microbenchmark",
         },
       },
-      subSectionRenders: {
-        detail_view: {
-          filterConstraint: {
-            model: {
-              disabled: true,
-            },
-            deviceName: {
-              disableOptions: [""],
-            },
-            mode: {
-              disableOptions: [""],
-            },
+    },
+    subSectionRenders: {
+      detail_view: {
+        filterConstraint: {
+          model: {
+            disabled: true,
           },
-          renders: [
-            {
-              type: "AutoBenchmarkTimeSeriesChartGroup",
-              title: "Metrics Time Series Chart Detail View",
-              config: {
-                type: "line",
-                groupByFields: ["metric"],
-                lineKey: ["extra_key.use_compile", "dtype", "metric", "branch"],
-                chart: {
-                  renderOptions: {
-                    chartRenderBook: RENDER_MAPPING_BOOK,
-                    showLegendDetails: true,
-                  },
-                },
-              },
-            },
-            {
-              type: "AutoBenchmarkTimeSeriesTable",
-              title: "Comparison Table Detail View",
-              config: {
-                primary: {
-                  fields: ["model"],
-                  displayName: "Model",
-                },
-                comparisonPolicy: {
-                  latency: LATENCY_POLICY,
-                },
-                extraMetadata: COMPARISON_TABLE_METADATA_COLUMNS,
+          deviceName: {
+            disableOptions: [""],
+          },
+          mode: {
+            disableOptions: [""],
+          },
+        },
+        renders: [
+          {
+            type: "AutoBenchmarkTimeSeriesChartGroup",
+            title: "Metrics Time Series Chart Detail View",
+            config: {
+              type: "line",
+              groupByFields: ["metric"],
+              lineKey: ["extra_key.use_compile", "dtype", "metric", "branch"],
+              chart: {
                 renderOptions: {
-                  tableRenderingBook: RENDER_MAPPING_BOOK,
-                  flex: {
-                    primary: 2,
-                  },
+                  chartRenderBook: RENDER_MAPPING_BOOK,
+                  showLegendDetails: true,
                 },
               },
             },
-            {
-              type: "AutoBenchmarkRawDataTable",
-              title: "Raw Data Table",
-              config: {
-                extraMetadata: [
-                  BRANCH_METADATA_COLUMN,
-                  ...COMPARISON_TABLE_METADATA_COLUMNS,
-                ],
-                renderOptions: {
-                  tableRenderingBook: RENDER_MAPPING_BOOK,
+          },
+          {
+            type: "AutoBenchmarkTimeSeriesTable",
+            title: "Comparison Table Detail View",
+            config: {
+              primary: {
+                fields: ["model"],
+                displayName: "Model",
+              },
+              comparisonPolicy: {
+                latency: LATENCY_POLICY,
+              },
+              extraMetadata: COMPARISON_TABLE_METADATA_COLUMNS,
+              renderOptions: {
+                tableRenderingBook: RENDER_MAPPING_BOOK,
+                flex: {
+                  primary: 2,
                 },
               },
             },
-          ],
-        },
-      },
-      renders: [
-        {
-          type: "AutoBenchmarkShortcutCardList",
-          title: "Operator Lists",
-          config: {
-            filters: ["operatorName"],
           },
-        },
-        {
-          type: "AutoBenchmarkComparisonGithubExternalLink",
-          title: "Github runs (external)",
-          description: "See original github runs for left and right runs",
-          config: {},
-        },
-        {
-          type: "AutoBenchmarkPairwiseTable",
-          title: "Comparison Table",
-          config: {
-            primary: {
-              fields: ["model"],
-              displayName: "Model",
-              navigation: {
-                type: "subSectionRender",
-                value: "detail_view",
-                applyFilterFields: ["model", "mode", "device", "arch", "dtype"],
+          {
+            type: "AutoBenchmarkRawDataTable",
+            title: "Raw Data Table",
+            config: {
+              extraMetadata: [
+                BRANCH_METADATA_COLUMN,
+                ...COMPARISON_TABLE_METADATA_COLUMNS,
+              ],
+              renderOptions: {
+                tableRenderingBook: RENDER_MAPPING_BOOK,
               },
             },
-            comparisonPolicy: {
-              latency: LATENCY_POLICY,
+          },
+        ],
+      },
+    },
+    renders: [
+      {
+        type: "AutoBenchmarkShortcutCardList",
+        title: "Operator Lists",
+        config: {
+          filters: ["operatorName"],
+        },
+      },
+      {
+        type: "AutoBenchmarkComparisonGithubExternalLink",
+        title: "Github runs (external)",
+        description: "See original github runs for left and right runs",
+        config: {},
+      },
+      {
+        type: "AutoBenchmarkPairwiseTable",
+        title: "Comparison Table",
+        config: {
+          primary: {
+            fields: ["model"],
+            displayName: "Model",
+            navigation: {
+              type: "subSectionRender",
+              value: "detail_view",
+              applyFilterFields: ["model", "mode", "device", "arch", "dtype"],
             },
-            extraMetadata: COMPARISON_TABLE_METADATA_COLUMNS,
-            renderOptions: {
-              tableRenderingBook: RENDER_MAPPING_BOOK,
-              flex: {
-                primary: 2,
-              },
+          },
+          comparisonPolicy: {
+            latency: LATENCY_POLICY,
+          },
+          extraMetadata: COMPARISON_TABLE_METADATA_COLUMNS,
+          renderOptions: {
+            tableRenderingBook: RENDER_MAPPING_BOOK,
+            flex: {
+              primary: 2,
             },
           },
         },
-      ],
-    },
-  };
+      },
+    ],
+  },
+};
diff --git a/torchci/components/benchmark_v3/configs/teams/vllm/config.ts b/torchci/components/benchmark_v3/configs/teams/vllm/config.ts
@@ -4,7 +4,7 @@ import {
   DEFAULT_DASHBOARD_BENCHMARK_INITIAL,
 } from "../defaults/default_dashboard_config";
 
-export const PYTORCH_VLLM_BENCHMARK_ID = "vllm_benchmark";
+export const VLLM_BENCHMARK_ID = "vllm_benchmark";
 
 const COMPARISON_TABLE_METADATA_COLUMNS = [
   {
@@ -37,15 +37,15 @@ const COMPARISON_TABLE_METADATA_COLUMNS = [
   },
 ] as const;
 
-export const PytorchVllmBenchmarkDashoboardConfig: BenchmarkUIConfig = {
-  benchmarkId: PYTORCH_VLLM_BENCHMARK_ID,
-  apiId: PYTORCH_VLLM_BENCHMARK_ID,
+export const VllmBenchmarkDashboardConfig: BenchmarkUIConfig = {
+  benchmarkId: VLLM_BENCHMARK_ID,
+  apiId: VLLM_BENCHMARK_ID,
   title: "VLLM V1 Benchmark",
   type: "dashboard",
   dataBinding: {
     initial: {
       ...DEFAULT_DASHBOARD_BENCHMARK_INITIAL,
-      benchmarkId: PYTORCH_VLLM_BENCHMARK_ID,
+      benchmarkId: VLLM_BENCHMARK_ID,
       lbranch: "main(main)",
       rbranch: "main(main)",
     },
diff --git a/torchci/components/benchmark_v3/configs/teams/vllm/pytorch_x_vllm_config.ts b/torchci/components/benchmark_v3/configs/teams/vllm/pytorch_x_vllm_config.ts
diff --git a/torchci/lib/benchmark/api_helper/backend/dataFetchers/fetchers.ts b/torchci/lib/benchmark/api_helper/backend/dataFetchers/fetchers.ts
