Skip to content

Commit b2d671e

Browse files
huydhnhuydhn
authored
Fix benchmark dashboard device/arch mapping (#7659)
We now store the detail information about the runner in the `arch` column. The dashboard needs to be updated accordingly. MPS results are showing up now, but I will submit a PR later to correctly set the device name to MPS there. At the moment, it's set to `arm64-cpu` with `arm` arch, which is not wrong but it's easy to confuse it with Linux `aarch64`. --------- Signed-off-by: Huy Do <huydhn@gmail.com>
1 parent 7ce6658 commit b2d671e

6 files changed

Lines changed: 60 additions & 61 deletions

File tree

‎torchci/components/benchmark/compilers/common.tsx‎

Lines changed: 21 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -58,28 +58,28 @@ export const HELP_LINK =
5858
export const DTYPES = ["amp", "float16", "bfloat16", "quant", "notset"];
5959

6060
export const DEFAULT_DEVICE_NAME = "cuda (h100)";
61-
export const DISPLAY_NAMES_TO_DEVICE_NAMES: { [k: string]: string } = {
62-
"cuda (a100)": "cuda",
63-
"cuda (h100)": "cuda",
64-
"cuda (b200)": "cuda",
65-
"cpu (x86_64)": "cpu",
66-
"cpu (x86_zen)": "cpu",
67-
"cpu (aarch64)": "cpu",
68-
"rocm (mi300x)": "rocm",
69-
mps: "mps",
70-
xpu: "xpu",
61+
export const DISPLAY_NAMES_TO_DEVICE_NAMES: { [k: string]: string[] } = {
62+
"cuda (a100)": ["cuda"],
63+
"cuda (h100)": ["cuda"],
64+
"cuda (b200)": ["cuda"],
65+
"cpu (x86_64)": ["cpu"],
66+
"cpu (x86_zen)": ["cpu"],
67+
"cpu (aarch64)": ["cpu", "arm64-cpu"],
68+
"rocm (mi325x)": ["rocm"],
69+
// TODO (huydhn): Fix this on the gather runner info script to set it to mps correctly
70+
mps: ["mps", "arm64-cpu"],
71+
xpu: ["xpu"],
7172
};
72-
export const DISPLAY_NAMES_TO_ARCH_NAMES: { [k: string]: string } = {
73-
"cuda (a100)": "a100",
74-
"cuda (h100)": "h100",
75-
"cuda (b200)": "b200",
76-
"cpu (x86_64)": "x86_64",
77-
"cpu (x86_zen)": "x86_zen",
78-
"cpu (aarch64)": "aarch64",
79-
// TODO (huydhn): Figure out a way to get the GPU name for ROCm
80-
"rocm (mi300x)": "",
81-
mps: "",
82-
xpu: "",
73+
export const DISPLAY_NAMES_TO_ARCH_NAMES: { [k: string]: string[] } = {
74+
"cuda (a100)": ["a100"],
75+
"cuda (h100)": ["h100"],
76+
"cuda (b200)": ["b200"],
77+
"cpu (x86_64)": ["Xeon_Platinum_8488C_48c"],
78+
"cpu (x86_zen)": ["AMD_EPYC_9R14_96c"],
79+
"cpu (aarch64)": ["aarch64"],
80+
"rocm (mi325x)": ["AMD Instinct MI325X"],
81+
mps: ["arm"],
82+
xpu: ["x86_64", "Intel(R) Data Center GPU Max 1100"],
8383
};
8484
export const DISPLAY_NAMES_TO_WORKFLOW_NAMES: { [k: string]: string } = {
8585
"cuda (a100)": "inductor-A100-perf-nightly",

‎torchci/components/benchmark_v3/configs/teams/compilers/CompilerSearchBarDropdowns.tsx‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,8 +44,8 @@ export function CompilerSearchBarDropdowns() {
4444
dtype={stagedFilters.deviceName ?? ""}
4545
setDType={(val: string) => {
4646
setStagedFilter("deviceName", val);
47-
setStagedFilter("device", DISPLAY_NAMES_TO_DEVICE_NAMES[val]);
48-
setStagedFilter("arch", DISPLAY_NAMES_TO_ARCH_NAMES[val]);
47+
setStagedFilter("device", DISPLAY_NAMES_TO_DEVICE_NAMES[val][0]);
48+
setStagedFilter("arch", DISPLAY_NAMES_TO_ARCH_NAMES[val][0]);
4949
}}
5050
dtypes={Object.keys(DISPLAY_NAMES_TO_DEVICE_NAMES)}
5151
label="Device"

‎torchci/components/benchmark_v3/configs/teams/compilers/config.ts‎

Lines changed: 3 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -197,22 +197,6 @@ const RENDER_MAPPING_BOOK = {
197197
},
198198
};
199199

200-
// converter function
201-
export function toQueryArch(device: string, arch: string) {
202-
if (arch === undefined) return [];
203-
if (!device) return [];
204-
switch (device) {
205-
case "rocm":
206-
if (arch === "mi300x" || arch == "") return ["mi300x", "mi325x"];
207-
return [arch];
208-
default:
209-
if (arch === "") {
210-
return [];
211-
}
212-
return [arch];
213-
}
214-
}
215-
216200
export const compilerQueryParameterConverter: QueryParameterConverter = (
217201
inputs: QueryParameterConverterInputs
218202
) => {
@@ -228,17 +212,16 @@ export const compilerQueryParameterConverter: QueryParameterConverter = (
228212

229213
let models = getModels(f.model);
230214

231-
const device = DISPLAY_NAMES_TO_DEVICE_NAMES[f.deviceName];
232-
const arch = DISPLAY_NAMES_TO_ARCH_NAMES[f.deviceName];
233-
const arches = toQueryArch(device, arch);
215+
const devices = DISPLAY_NAMES_TO_DEVICE_NAMES[f.deviceName];
216+
const arches = DISPLAY_NAMES_TO_ARCH_NAMES[f.deviceName];
234217

235218
const params = {
236219
commits: i.commits ?? [],
237220
branches: i.branches ?? [],
238221
workflows: workflows,
239222
compilers: compilerList,
240223
arches: arches,
241-
devices: [device],
224+
devices: devices,
242225
dtypes: f.dtype === "none" ? [] : [f.dtype],
243226
granularity: "hour",
244227
modes: [f.mode],

‎torchci/lib/benchmark/api_helper/backend/compilers/compiler_benchmark_data.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ import {
88
getCompilerCommitsWithSampling,
99
} from "../common/utils";
1010
import { BenchmarkCompilerBenchmarkDataQueryBuilder } from "../dataFetchers/queryBuilderUtils/compilerQueryBuilder";
11-
import { extractBackendSqlStyle, toApiArch } from "./helpers/common";
11+
import { extractBackendSqlStyle, toApiDeviceArch } from "./helpers/common";
1212
import { toGeneralCompilerData } from "./helpers/general";
1313
import { toPrecomputeCompilerData } from "./helpers/precompute";
1414

@@ -148,6 +148,7 @@ async function fetchCompilerDataFromDb(queryParams: any): Promise<any[]> {
148148
// extract backend from output in runtime instead of doing it in the query. since it's expensive for regex matching.
149149
// TODO(elainewy): we should add this as a column in the database for less runtime logics.
150150
rows.map((row) => {
151+
[row["device"], row["arch"]] = toApiDeviceArch(row.device, row.arch);
151152
const backend =
152153
row.backend && row.backend !== ""
153154
? row.backend
@@ -159,7 +160,6 @@ async function fetchCompilerDataFromDb(queryParams: any): Promise<any[]> {
159160
row.device
160161
);
161162
(row["backend"] = backend), (row["compiler"] = backend);
162-
row["arch"] = toApiArch(row.device, row.arch);
163163
});
164164

165165
if (queryParams.compilers && queryParams.compilers.length > 0) {

‎torchci/lib/benchmark/api_helper/backend/compilers/helpers/common.ts‎

Lines changed: 28 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -14,25 +14,41 @@ export function extractBackendSqlStyle(
1414
return m ? m[1] : null;
1515
}
1616

17-
export function toApiArch(device: string, arch: string): string {
17+
export function toApiDeviceArch(
18+
device: string,
19+
arch: string
20+
): [string, string] {
1821
const norm = arch.toLowerCase();
22+
// TODO (huydhn): Clean this up once the new name has been around for some time
1923
switch (device) {
2024
case "cpu":
21-
return norm;
25+
if (norm.includes("xeon") || norm.includes("x86_64"))
26+
return [device, "x86_64"];
27+
if (norm.includes("amd") || norm.includes("x86_zen"))
28+
return [device, "x86_zen"];
29+
return [device, norm];
30+
case "arm64-cpu":
31+
if (norm.includes("aarch64")) return ["cpu", norm];
32+
if (norm.includes("arm")) return ["mps", norm];
33+
return [device, norm];
2234
case "cuda":
23-
if (norm.includes("h100")) return "h100";
24-
if (norm.includes("a100")) return "a100";
25-
if (norm.includes("a10g")) return "a10g";
26-
if (norm.includes("b200")) return "b200";
27-
return norm;
35+
if (norm.includes("h100")) return [device, "h100"];
36+
if (norm.includes("a100")) return [device, "a100"];
37+
if (norm.includes("a10g")) return [device, "a10g"];
38+
if (norm.includes("b200")) return [device, "b200"];
39+
return [device, norm];
2840
case "rocm":
29-
if (norm.includes("mi300x")) return "mi300x";
30-
if (norm.includes("mi325x")) return "mi300x";
31-
return norm;
41+
if (norm.includes("mi300x")) return [device, "mi300x"];
42+
if (norm.includes("mi325x")) return [device, "mi325x"];
43+
return [device, norm];
3244
case "mps":
33-
return norm;
45+
return [device, norm];
46+
case "xpu":
47+
if (norm.includes("intel") || norm.includes("x86_64"))
48+
return [device, "x86_64"];
49+
return [device, norm];
3450
default:
35-
return norm;
51+
return [device, norm];
3652
}
3753
}
3854

‎torchci/lib/benchmark/api_helper/backend/compilers/helpers/precompute.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ import {
1515
toTimeSeriesResponse,
1616
toWorkflowIdMap,
1717
} from "../../common/utils";
18-
import { toApiArch } from "./common";
18+
import { toApiDeviceArch } from "./common";
1919

2020
const COMPILER_PRECOMPUTE_TS_GROUP_KEY = [
2121
"dtype",
@@ -166,15 +166,15 @@ export function groupByBenchmark(rawData: any[]) {
166166
const groups: Record<string, any[]> = {};
167167
const metadataMapping: Record<string, any> = {};
168168
for (const item of rawData) {
169-
const apiArch = toApiArch(item.device, item.arch);
169+
const [apiDevice, apiArch] = toApiDeviceArch(item.device, item.arch);
170170
// composite grouping key
171-
const key = `${apiArch}_${item.device}_${item.dtype}_${item.mode}`;
171+
const key = `${apiArch}_${apiDevice}_${item.dtype}_${item.mode}`;
172172
if (!metadataMapping[key]) {
173173
metadataMapping[key] = {
174174
dtype: item.dtype,
175175
arch: apiArch,
176176
mode: item.mode,
177-
device: item.device,
177+
device: apiDevice,
178178
};
179179
}
180180
if (!groups[key]) {

0 commit comments

Comments
 (0)