[HUD] SGLang vs vLLM Comparison Dashboard enhancements (#7135)

## Changes
This PR includes the following changes in the comparison dashboard:

- Hyperlink issue fix - Currently the hyperlink for the comparison
dashboard was not working correctly, as it was redirecting to some other
benchmark.
- Adding a new dropdown option of `QPS` (Request rate), where defaults
to `All QPS` and contains options which are dynamically fetched from the
"model"'s benchmark data fetched from the clickhouse database.
- Adding implementation logic for filtering the comparison data, based
on the selected QPS before rendering graphs.
- In the "Data details" table in the dashboard, once a model is
selected, the "commit" column was empty. Fixed that bug to fetch the
correct commit hash and display in the table.
- Make sure that once a QPS is selected, all the corresponding data in
that page (including the tables) is correctly populated based on that
QPS only.
- Whenever we visit the comparison dashboard, and no model is selected,
we were earlier showing all the models present in both the repositories,
but ideally we should show only those models which have data common to
both SGLang and vLLM repos. Fixed that issue with these changes.

## Testing
- Verified from the deployed dashboard:
[link](https://torchci-d5hd5oow3-fbopensource.vercel.app/benchmark/llms?repos=vllm-project%2Fvllm%2Csgl-project%2Fsglang)

- Hyperlink is working fine now in both cases (when no model is selected
vs when a model is selected)
- With model:
[Hyperlink](https://torchci-d5hd5oow3-fbopensource.vercel.app/benchmark/llms?startTime=Wed%2C%2003%20Sep%202025%2006%3A58%3A42%20GMT&stopTime=Wed%2C%2010%20Sep%202025%2006%3A58%3A42%20GMT&granularity=day&lBranch=main&lCommit=&rBranch=main&rCommit=&repoName=pytorch%2Fpytorch&benchmarkName=&modelName=meta-llama%2FMeta-Llama-3.1-8B-Instruct&backendName=All%20Backends&modeName=All%20Modes&dtypeName=All%20DType&deviceName=All%20Devices&archName=All%20Platforms&repos=vllm-project%2Fvllm%2Csgl-project%2Fsglang&qps=All%20QPS)
- Without model:
[Hyperlink](https://torchci-d5hd5oow3-fbopensource.vercel.app/benchmark/llms?startTime=Wed%2C%2003%20Sep%202025%2006%3A58%3A42%20GMT&stopTime=Wed%2C%2010%20Sep%202025%2006%3A58%3A42%20GMT&granularity=day&lBranch=main&lCommit=&rBranch=main&rCommit=&repoName=pytorch%2Fpytorch&benchmarkName=&modelName=All%20Models&backendName=All%20Backends&modeName=All%20Modes&dtypeName=All%20DType&deviceName=All%20Devices&archName=All%20Platforms&repos=vllm-project%2Fvllm%2Csgl-project%2Fsglang&qps=All%20QPS)


- Main comparison page showing only common models entries.
<img width="1704" height="894" alt="Screenshot 2025-09-09 at 11 12
47 PM"
src="https://github.com/user-attachments/assets/b577b828-6585-4ef5-bb8e-2b6d4017f77b"
/>

- QPS Filter enabled when a model is selected, and showing qps values
that are fetched dynamically for that particular model.
<img width="1704" height="894" alt="Screenshot 2025-09-09 at 11 13
57 PM"
src="https://github.com/user-attachments/assets/670ee1e8-e01c-4cc6-aa07-29bf541b88a2"
/>

- When a certain QPS Filter is selected, then only those records should
be rendered in that dashboard page.
<img width="1704" height="894" alt="Screenshot 2025-09-09 at 11 15
09 PM"
src="https://github.com/user-attachments/assets/d20248d0-eadb-4f83-ae73-8275d65d7f92"
/>

- Data details table showing the "commit" column correctly.
<img width="1704" height="861" alt="Screenshot 2025-09-09 at 11 15
30 PM"
src="https://github.com/user-attachments/assets/d0a14409-5181-45ff-8384-20857af726df"
/>

Author: namanlalitnyu

torchci/clickhouse_queries/oss_ci_benchmark_llms/params.json

@@ -13,7 +13,8 @@
     "backends": "Array(String)",
     "repo": "String",
     "startTime": "DateTime64(3)",
-    "stopTime": "DateTime64(3)"
+    "stopTime": "DateTime64(3)",
+    "requestRate": "String"
   },
   "tests": [
     {
@@ -30,7 +31,8 @@
       "backends": [],
       "repo": "pytorch/pytorch",
       "startTime": "2025-01-28T22:46:18.214",
-      "stopTime": "2025-02-04T22:46:18.214"
+      "stopTime": "2025-02-04T22:46:18.214",
+      "requestRate": ""
     }
   ]
 }

torchci/clickhouse_queries/oss_ci_benchmark_llms/query.sql

@@ -133,6 +133,13 @@ WITH benchmarks AS (
             NOT has({excludedMetrics: Array(String) }, o.metric.'name')
             OR empty({excludedMetrics: Array(String) })
         )
+        AND (
+            JSONExtractString(
+                tupleElement(o.benchmark, 'extra_info')['args'],
+                'request_rate'
+            ) = {requestRate: String}
+            OR {requestRate: String} = ''
+        )
         AND notEmpty(o.metric.'name')
 )
 

torchci/components/benchmark/llms/LLMsBenchmarkPage.tsx

@@ -21,6 +21,7 @@ import {
   DEFAULT_DTYPE_NAME,
   DEFAULT_MODE_NAME,
   DEFAULT_MODEL_NAME,
+  DEFAULT_QPS_NAME,
   HELION_BENCHMARK_NAME,
   REPO_TO_BENCHMARKS,
 } from "lib/benchmark/llms/common";
@@ -52,6 +53,7 @@ export default function LLMsBenchmarkPage() {
     dtypeName: DEFAULT_DTYPE_NAME,
     deviceName: DEFAULT_DEVICE_NAME,
     archName: DEFAULT_ARCH_NAME,
+    qps: DEFAULT_QPS_NAME,
     startTime: defaultStartTime,
     stopTime: defaultStopTime,
     timeRange: LAST_N_DAYS,

torchci/components/benchmark/llms/LLMsComparingBenchmarkPage.tsx

@@ -21,16 +21,19 @@ import {
   DEFAULT_DTYPE_NAME,
   DEFAULT_MODE_NAME,
   DEFAULT_MODEL_NAME,
+  DEFAULT_QPS_NAME,
   HELION_BENCHMARK_NAME,
   LLM_BENCHMARK_CONFIG_QUERY,
+  LLM_BENCHMARK_DATA_QUERY,
   REPO_TO_BENCHMARKS,
 } from "lib/benchmark/llms/common";
 import { LLMsBenchmarkMode } from "lib/benchmark/llms/types/benchmarkMode";
+import { DropdownGroupItemType } from "lib/benchmark/llms/types/dashboardPickerTypes";
 import { LLMsBenchmarkProps } from "lib/benchmark/llms/types/dashboardProps";
 import { getBenchmarkDropdownFeatures } from "lib/benchmark/llms/utils/dashboardPickerUtils";
 import {
-  fetchBenchmarkDataForRepos,
   getLLMsBenchmarkPropsQueryParameter,
+  useBenchmarkDataForRepos,
 } from "lib/benchmark/llms/utils/llmUtils";
 import { LLMsDashboardPicker } from "./components/dashboardPicker/LLMsDashboardPicker";
 import { LLMsTimeRangePicker } from "./components/dashboardPicker/LLMsTimeRangePicker";
@@ -51,6 +54,7 @@ export default function LLMsComparingBenchmarkPage() {
     dtypeName: DEFAULT_DTYPE_NAME,
     deviceName: DEFAULT_DEVICE_NAME,
     archName: DEFAULT_ARCH_NAME,
+    qps: DEFAULT_QPS_NAME,
     startTime: defaultStartTime,
     stopTime: defaultStopTime,
     timeRange: LAST_N_DAYS,
@@ -114,14 +118,46 @@ const MainPageForComparison = ({
   dispatch: React.Dispatch<any>;
 }) => {
   const [baseUrl, setBaseUrl] = useState<string>("");
-  const [allRepoData, setAllRepoData] = useState<any[]>([]);
-  const [allRepoErrors, setAllRepoErrors] = useState<any[]>([]);
-  const [isLoading, setIsLoading] = useState<boolean>(true);
+  const [qpsOptions, setQpsOptions] = useState<string[]>([]);
+  const [modelQpsMap, setModelQpsMap] = useState<Record<string, string[]>>({});
   const queryParams = useMemo(
     () => getLLMsBenchmarkPropsQueryParameter(props),
     [props]
   );
 
+  const repoQueryParams = useMemo(
+    () =>
+      props.repos.map((repo) => {
+        const repoSpecificProps = { ...props, repoName: repo, repos: [] };
+        return getLLMsBenchmarkPropsQueryParameter(repoSpecificProps);
+      }),
+    [
+      props.repos,
+      props.benchmarkName,
+      props.modelName,
+      props.backendName,
+      props.modeName,
+      props.dtypeName,
+      props.deviceName,
+      props.archName,
+      props.startTime,
+      props.stopTime,
+      props.granularity,
+      props.lCommit,
+      props.rCommit,
+      props.lBranch,
+      props.rBranch,
+    ]
+  );
+
+  const { data: configResults } = useBenchmarkDataForRepos(
+    LLM_BENCHMARK_CONFIG_QUERY,
+    repoQueryParams
+  );
+  const allRepoData = configResults?.map((r: any) => r.data) || [];
+  const allRepoErrors = configResults?.map((r: any) => r.error) || [];
+  const isLoading = !configResults;
+
   useEffect(() => {
     const newProps = resetProps(
       router.query,
@@ -137,44 +173,101 @@ const MainPageForComparison = ({
     );
   }, [router.query]);
 
+  // Data fetching handled by useBenchmarkDataForRepos
+
+  const modelQpsQueryParams = useMemo(
+    () =>
+      props.repos.map((repo) => {
+        const repoSpecificProps = {
+          ...props,
+          repoName: repo,
+          repos: [],
+          modelName: DEFAULT_MODEL_NAME,
+          qps: DEFAULT_QPS_NAME,
+        };
+        return getLLMsBenchmarkPropsQueryParameter(repoSpecificProps);
+      }),
+    [
+      props.repos,
+      props.backendName,
+      props.modeName,
+      props.dtypeName,
+      props.deviceName,
+      props.archName,
+      props.startTime,
+      props.stopTime,
+      props.benchmarkName,
+      props.granularity,
+      props.rBranch,
+      props.rCommit,
+    ]
+  );
+
+  const modelQpsParamsWithBranch = useMemo(
+    () =>
+      modelQpsQueryParams.map((qp) => ({
+        ...qp,
+        branches: props.rBranch ? [props.rBranch] : [],
+        commits: props.rCommit ? [props.rCommit] : [],
+      })),
+    [modelQpsQueryParams, props.rBranch, props.rCommit]
+  );
+
+  const { data: modelQpsResults } = useBenchmarkDataForRepos(
+    LLM_BENCHMARK_DATA_QUERY,
+    modelQpsParamsWithBranch
+  );
+
   useEffect(() => {
-    let cancelled = false;
-    const repoQueryParams = props.repos.map((repo) => {
-      const repoSpecificProps = { ...props, repoName: repo };
-      return getLLMsBenchmarkPropsQueryParameter(repoSpecificProps);
-    });
-    setIsLoading(true);
-    fetchBenchmarkDataForRepos(
-      LLM_BENCHMARK_CONFIG_QUERY,
-      repoQueryParams
-    ).then((results) => {
-      if (cancelled) {
-        return;
-      }
-      setAllRepoData(results.map((r) => r.data));
-      setAllRepoErrors(results.map((r) => r.error));
-      setIsLoading(false);
+    if (!modelQpsResults) {
+      return;
+    }
+    const map: Record<string, string[]> = {};
+    modelQpsResults.forEach((r: any) => {
+      const data = (r.data || []) as any[];
+      const grouped = _.groupBy(data, (rec) => rec.model);
+      Object.entries(grouped).forEach(([model, recs]) => {
+        const qpsValues = _.uniq(
+          recs
+            .map((rec: any) => rec.extra?.request_rate)
+            .filter(
+              (v): v is string | number =>
+                v !== undefined &&
+                v !== null &&
+                v !== "" &&
+                (v === "inf" || !isNaN(Number(v)))
+            )
+            .map((v) => (v === "inf" ? "inf" : String(Number(v))))
+        );
+        map[model] = _.uniq([...(map[model] || []), ...qpsValues]);
+      });
     });
-    return () => {
-      cancelled = true;
-    };
-  }, [
-    props.repos,
-    props.benchmarkName,
-    props.modelName,
-    props.backendName,
-    props.modeName,
-    props.dtypeName,
-    props.deviceName,
-    props.archName,
-    props.startTime,
-    props.stopTime,
-    props.granularity,
-    props.lCommit,
-    props.rCommit,
-    props.lBranch,
-    props.rBranch,
-  ]);
+    Object.keys(map).forEach((m) =>
+      map[m].sort(
+        (a, b) =>
+          (a === "inf" ? Infinity : Number(a)) -
+          (b === "inf" ? Infinity : Number(b))
+      )
+    );
+    setModelQpsMap(map);
+  }, [modelQpsResults]);
+
+  useEffect(() => {
+    if (props.modelName === DEFAULT_MODEL_NAME) {
+      setQpsOptions([]);
+      dispatch({ type: "UPDATE_FIELD", field: "qps", value: DEFAULT_QPS_NAME });
+      return;
+    }
+    const shared = modelQpsMap[props.modelName] || [];
+    setQpsOptions([DEFAULT_QPS_NAME, ...shared]);
+    if (!shared.includes(props.qps)) {
+      dispatch({
+        type: "UPDATE_FIELD",
+        field: "qps",
+        value: DEFAULT_QPS_NAME,
+      });
+    }
+  }, [props.modelName, modelQpsMap]);
 
   const hasError = allRepoErrors.some((error) => error);
   if (hasError) {
@@ -223,7 +316,7 @@ const MainPageForComparison = ({
     );
   }
 
-  const combinedData = allRepoData.flatMap((repoData, index) => {
+  let combinedData = allRepoData.flatMap((repoData, index) => {
     const repo = props.repos[index];
     return repoData.map((dataItem: any) => ({
       ...dataItem,
@@ -255,6 +348,26 @@ const MainPageForComparison = ({
     };
   });
 
+  if (qpsOptions.length > 1) {
+    dropdownMapList.push({
+      type: DropdownGroupItemType.Qps,
+      labelName: "QPS",
+      options: qpsOptions,
+    });
+  }
+
+  if (props.modelName === DEFAULT_MODEL_NAME) {
+    const modelDropdown = dropdownMapList.find(
+      (d) => d.type === DropdownGroupItemType.ModelName
+    );
+    if (modelDropdown) {
+      const sharedModels = modelDropdown.options.slice(1);
+      combinedData = combinedData.filter((d: any) =>
+        sharedModels.includes(d.model)
+      );
+    }
+  }
+
   const metricNames = getMetricNames(combinedData);
   // Default to latest for Helion Benchmark, otherwise default to oldest commit
   const lcommitFallbackIdx =
@@ -364,6 +477,11 @@ function resetProps(
     newProps.archName = archName;
   }
 
+  const qps: string = (urlQuery.qps as string) ?? undefined;
+  if (qps !== undefined) {
+    newProps.qps = qps;
+  }
+
   const lBranch: string = (urlQuery.lBranch as string) ?? undefined;
   if (lBranch !== undefined) {
     newProps.lBranch = lBranch;
@@ -435,7 +553,11 @@ const formLink = (props: LLMsBenchmarkProps, baseUrl: string) => {
         props.dtypeName
       )}&deviceName=${encodeURIComponent(
         props.deviceName
-      )}&archName=${encodeURIComponent(props.archName)}`}
+      )}&archName=${encodeURIComponent(
+        props.archName
+      )}&repos=${encodeURIComponent(
+        props.repos.join(",")
+      )}&qps=${encodeURIComponent(props.qps)}`}
     />
   );
 };