[Add API] get_time_series

.gitignore

@@ -69,3 +69,6 @@ aws/tools/cleanup-ssm/Cargo.lock
 
 # These are backup files generated by rustfmt
 aws/tools/cleanup-ssm/**/*.rs.bk
+
+# Remove the python version file from pyenv
+.python-version

torchci/clickhouse_queries/compilers_benchmark_performance_branches/query.sql

@@ -7,7 +7,7 @@ SELECT DISTINCT
     toStartOfDay(fromUnixTimestamp(timestamp)) AS event_time
 FROM
     benchmark.oss_ci_benchmark_torchinductor
-WHERE
+PREWHERE
     timestamp >= toUnixTimestamp({startTime: DateTime64(3) })
     AND timestamp < toUnixTimestamp({stopTime: DateTime64(3) })
     -- TODO (huydhn): Clean up the output field and how it's used in the query

torchci/components/benchmark/compilers/SummaryGraphPanel.tsx

@@ -235,6 +235,7 @@ function SuiteGraphPanel({
       (id <= lWorkflowId && id >= rWorkflowId)
     );
   });
+
   const peakMemorySeries = seriesWithInterpolatedTimes(
     peakMemory,
     startTime,

torchci/lib/benchmark/api_helper/compilers/precompute.ts

@@ -0,0 +1,173 @@
+import {
+  computeGeomean,
+  computeMemoryCompressionRatio,
+  computePassrate,
+  convertToCompilerPerformanceData,
+  getPassingModels,
+} from "lib/benchmark/compilerUtils";
+import { queryClickhouseSaved } from "lib/clickhouse";
+import {
+  BenchmarkTimeSeriesResponse,
+  CommitRow,
+  groupByBenchmarkData,
+  toCommitRowMap,
+} from "../utils";
+
+const BENCNMARK_TABLE_NAME = "compilers_benchmark_performance";
+const BENCNMARK_COMMIT_NAME = "compilers_benchmark_performance_branches";
+
+// TODO(elainewy): improve the fetch performance
+export async function getCompilerBenchmarkData(inputparams: any) {
+  const start = Date.now();
+  const rows = await queryClickhouseSaved(BENCNMARK_TABLE_NAME, inputparams);
+  const end = Date.now();
+  console.log("time to get data", end - start);
+
+  const startc = Date.now();
+  const commits = await queryClickhouseSaved(
+    BENCNMARK_COMMIT_NAME,
+    inputparams
+  );
+  const endc = Date.now();
+  console.log("time to get commit data", endc - startc);
+  const commitMap = toCommitRowMap(commits);
+
+  if (rows.length === 0) {
+    const response: BenchmarkTimeSeriesResponse = {
+      time_series: [],
+      time_range: {
+        start: "",
+        end: "",
+      },
+    };
+    return response;
+  }
+
+  // TODO(elainewy): add logics to handle the case to return raw data
+  const benchmark_time_series_response = toPrecomputeCompiler(
+    rows,
+    inputparams,
+    commitMap,
+    "time_series"
+  );
+  return benchmark_time_series_response;
+}
+
+function toPrecomputeCompiler(
+  rawData: any[],
+  inputparams: any,
+  commitMap: Record<string, CommitRow>,
+  type: string = "time_series"
+) {
+  const data = convertToCompilerPerformanceData(rawData);
+  const models = getPassingModels(data);
+
+  const passrate = computePassrate(data, models);
+  const geomean = computeGeomean(data, models);
+  const peakMemory = computeMemoryCompressionRatio(data, models);
+
+  const all_data = [passrate, geomean, peakMemory].flat();
+
+  const earliest_timestamp = Math.min(
+    ...all_data.map((row) => new Date(row.granularity_bucket).getTime())
+  );
+  const latest_timestamp = Math.max(
+    ...all_data.map((row) => new Date(row.granularity_bucket).getTime())
+  );
+
+  //TODO(elainewy): remove this after change the schema of compiler database to populate the fields directly
+  all_data.map((row) => {
+    row["dtype"] = inputparams["dtype"];
+    row["arch"] = inputparams["arch"];
+    row["device"] = inputparams["device"];
+    row["mode"] = inputparams["mode"];
+    // always keep this:
+    row["commit"] = commitMap[row["workflow_id"]]?.head_sha;
+    row["branch"] = commitMap[row["workflow_id"]]?.head_branch;
+  });
+
+  let res: any[] = [];
+  switch (type) {
+    case "time_series":
+      /**
+       * Response of groupByBenchmarkData:
+       * [
+       *   {
+       *     "group_info": {
+       *       "dtype": "fp32",
+       *       "arch": "sm80",
+       *       "device": "cuda",
+       *       "suite": "ads_10x",
+       *       "compiler": "gcc9.3.0",
+       *       "metric": "latency",
+       *       "mode": "eager"
+       *     },
+       *     "rows": [
+       *        "f123456": {
+       *          "group_info": {
+       *           "workflow_id": "f123456"
+       *          },
+       *          "data": [ # list of data that has the same group_info for group keys and sub group keys
+       *           {
+       *             "workflow_id": "f123456",
+       *             "granularity_bucket": "2022-10-01 00:00:00",
+       *             "value": 100
+       *             ...
+       *           }
+       *         ],
+       *       },
+       *     ]
+       *   }
+       * ]
+       */
+      const tsd = groupByBenchmarkData(
+        all_data,
+        ["dtype", "arch", "device", "suite", "compiler", "metric", "mode"],
+        ["workflow_id"]
+      );
+
+      res = tsd.map((group) => {
+        const group_info = group.group_Info;
+        const sub_group_data = group.rows;
+        // extract the first data point for each sub group
+        // since we only have one datapoint for each unique workflow id with the same group info
+        const ts_list = Object.values(sub_group_data)
+          .filter((item) => item.data.length > 0)
+          .map((item) => item.data[0])
+          .sort(
+            (a, b) =>
+              new Date(a.granularity_bucket).getTime() -
+              new Date(b.granularity_bucket).getTime()
+          );
+        return {
+          group_info,
+          num_of_dp: ts_list.length,
+          data: ts_list,
+        };
+      });
+      break;
+    case "table":
+      res = groupByBenchmarkData(
+        all_data,
+        [
+          "dtype",
+          "arch",
+          "device",
+          "mode",
+          "workflow_id",
+          "granularity_bucket",
+        ],
+        ["metric", "compiler"]
+      );
+      break;
+  }
+
+  const response: BenchmarkTimeSeriesResponse = {
+    time_series: res,
+    time_range: {
+      start: new Date(earliest_timestamp).toISOString(),
+      end: new Date(latest_timestamp).toISOString(),
+    },
+  };
+  return response;
+}

torchci/lib/benchmark/api_helper/utils.ts

@@ -0,0 +1,155 @@
+// Utility to extract params from either GET or POST
+
+import { NextApiRequest } from "next";
+
+/**
+ * Key-value map describing metadata for a group.
+ * Example: { dtype: "fp32", arch: "sm80", device: "cuda" }
+ */
+type GroupInfo = Record<string, string>;
+
+/**
+ * Represents a subgroup within a larger group.
+ * Contains its own metadata and a list of data items.
+ */
+type Subgroup<T> = {
+  /** Metadata fields for this subgroup (e.g., workflow_id). */
+  group_info: GroupInfo;
+
+  /** The actual list of data items belonging to this subgroup. */
+  data: T[];
+};
+
+/**
+ * Represents a grouped item at the top level.
+ * Contains group-level metadata and a collection of subgroups.
+ */
+type GroupedItem<T> = {
+  /** Metadata fields for this group (e.g., dtype, arch, compiler). */
+  group_Info: GroupInfo;
+
+  /**
+   * Rows keyed by a unique identifier string,
+   * derived from a distinct combination of subgroup `group_Info` fields.
+   * Each entry corresponds to one subgroup that contains data points.
+   */
+  rows: Record<string, Subgroup<T>>;
+};
+
+/**
+ * Generic parameters map passed into functions or queries.
+ * Example: { startTime: "2025-08-24", device: "cuda", arch: "h100" }
+ */
+type Params = Record<string, any>;
+
+// it accepts both ?parameters=<json string> and POST with JSON body
+export function readApiGetParams(req: NextApiRequest): Params {
+  // 1) If POST with parsed JSON body
+  if (req.method === "POST" && req.body && typeof req.body === "object") {
+    return req.body as Params;
+  }
+
+  // 2) If POST with raw string body
+  if (
+    req.method === "POST" &&
+    typeof req.body === "string" &&
+    req.body.trim()
+  ) {
+    try {
+      return JSON.parse(req.body) as Params;
+    } catch {}
+  }
+
+  // 3) If GET with ?parameters=<json string>
+  const raw = req.query.parameters as string | undefined;
+  if (raw) {
+    try {
+      return JSON.parse(raw) as Params;
+    } catch {}
+  }
+
+  // 4) Fallback: use query params directly
+  const q: Params = {};
+  Object.entries(req.query).forEach(([k, v]) => {
+    if (k !== "parameters") q[k] = Array.isArray(v) ? v[0] : v;
+  });
+  return q;
+}
+
+/**
+ * Group benchmark data by `keys`, and inside each group further subgroup by `subGroupKeys`.
+ * @param data - benchmark data
+ * @param keys - keys to group by
+ * @param subGroupKeys - keys to subgroup by (optional): if not provided, a single subgroup will be created with "_ALL_" data
+ */
+export function groupByBenchmarkData<T>(
+  data: T[],
+  keys: string[],
+  subGroupKeys: string[] = []
+): GroupedItem<T>[] {
+  const groups = new Map<string, Map<string, Subgroup<T>>>();
+  const mainInfo = new Map<string, GroupInfo>();
+
+  for (const row of data as any[]) {
+    // build main group key
+    const mainKeyParts = keys.map((k) => String(getNestedField(row, k)));
+    const mainKey = mainKeyParts.join("|");
+    if (!mainInfo.has(mainKey)) {
+      const info: GroupInfo = {};
+      keys.forEach((k, i) => (info[k] = mainKeyParts[i]));
+      mainInfo.set(mainKey, info);
+    }
+
+    // build subgroup key
+    const subKeyParts =
+      subGroupKeys.length > 0
+        ? subGroupKeys.map((k) => String(getNestedField(row, k)))
+        : ["__ALL__"]; // default single subgroup if none provided
+    const subKey = subKeyParts.join("|");
+    const subInfo: GroupInfo = {};
+
+    subGroupKeys.forEach((k, i) => (subInfo[k] = subKeyParts[i]));
+
+    if (!groups.has(mainKey)) groups.set(mainKey, new Map());
+    const subMap = groups.get(mainKey)!;
+
+    if (!subMap.has(subKey)) {
+      subMap.set(subKey, { group_info: subInfo, data: [] });
+    }
+    subMap.get(subKey)!.data.push(row as T);
+  }
+
+  // build result array
+  const result: GroupedItem<T>[] = [];
+  for (const [mainKey, subMap] of groups.entries()) {
+    const rowsObj = Object.fromEntries(subMap.entries());
+    result.push({
+      group_Info: mainInfo.get(mainKey)!,
+      rows: rowsObj,
+    });
+  }
+  return result;
+}
+
+export function getNestedField(obj: any, path: string): any {
+  return path.split(".").reduce((o, key) => (o && key in o ? o[key] : ""), obj);
+}
+
+export type BenchmarkTimeSeriesResponse = {
+  time_series: any[];
+  time_range: { start: string; end: string };
+};
+
+export type CommitRow = {
+  head_branch: string;
+  head_sha: string;
+  id: string;
+};
+
+export function toCommitRowMap(rows: CommitRow[]): Record<string, CommitRow> {
+  const result: Record<string, CommitRow> = {};
+  for (const row of rows) {
+    result[row.id] = row;
+  }
+  return result;
+}

torchci/lib/benchmark/compilerUtils.ts

@@ -99,6 +99,8 @@ export function computePassrate(
 
     const [bucket, workflowId, suite, compiler] = key.split("+");
     passrate.push({
+      metric: "passrate",
+      value: p,
       granularity_bucket: bucket,
       workflow_id: workflowId,
       suite: suite,
@@ -163,6 +165,8 @@ export function computeGeomean(
 
     const [bucket, workflowId, suite, compiler] = key.split("+");
     returnedGeomean.push({
+      metric: "geomean",
+      value: Number(gm),
       granularity_bucket: bucket,
       workflow_id: workflowId,
       suite: suite,
@@ -274,6 +278,7 @@ export function computeCompilationTime(
 
     const [bucket, workflowId, suite, compiler] = key.split("+");
     returnedCompTime.push({
+      metric: "compilation_latency",
       granularity_bucket: bucket,
       workflow_id: workflowId,
       suite: suite,
@@ -328,6 +333,8 @@ export function computeMemoryCompressionRatio(
 
     const [bucket, workflowId, suite, compiler] = key.split("+");
     returnedMemory.push({
+      metric: "compression_ratio",
+      value: Number(m.toFixed(SCALE)),
       granularity_bucket: bucket,
       workflow_id: workflowId,
       suite: suite,
@@ -379,6 +386,8 @@ export function computePeakMemoryUsage(
 
     const [bucket, workflowId, suite, compiler] = key.split("+");
     returnedMemory.push({
+      metric: "dynamo_peak_mem",
+      value: Number(m.toFixed(SCALE)),
       granularity_bucket: bucket,
       workflow_id: workflowId,
       suite: suite,