Skip to content

Commit 81112ff

Browse files
joseph-isaacsjoseph-isaacs
authored
chore[benchmark]: add scale factor 100 (#3751)
support scale factor 100 in benchmark website --------- Signed-off-by: Joe Isaacs <joe.isaacs@live.co.uk>
1 parent 7b04494 commit 81112ff

File tree

4 files changed

+107
-36
lines changed

4 files changed

+107
-36
lines changed

‎.github/workflows/nightly-bench.yml‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
name: Nightly Benchmarks
2+
3+
on:
4+
schedule:
5+
- cron: '0 0 * * *' # daily
6+
7+
permissions:
8+
id-token: write # enables AWS-GitHub OIDC
9+
actions: read
10+
contents: write
11+
deployments: write
12+
13+
jobs:
14+
sql:
15+
uses: ./.github/workflows/sql-benchmarks.yml
16+
secrets: inherit
17+
with:
18+
mode: 'develop'
19+
machine_type: ${{ matrix.machine_type }}
20+
scale_factor: 100
21+
strategy:
22+
matrix:
23+
machine_type:
24+
- id: x86
25+
name: c6id.8xlarge
26+
# TODO(joe): support other arch
27+
# - id: arm64
28+
# name: c6gd.8xlarge

‎.github/workflows/sql-benchmarks.yml‎

Lines changed: 20 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,14 @@ on:
66
mode:
77
required: true
88
type: string
9+
machine_type:
10+
required: false
11+
type: string
12+
default: c6id.8xlarge
13+
scale_factor:
14+
required: false
15+
type: number
16+
default: 1
917

1018
jobs:
1119
bench:
@@ -23,22 +31,24 @@ jobs:
2331
- id: tpch-nvme
2432
binary_name: tpch
2533
name: TPC-H on NVME
26-
local_dir: bench-vortex/data/tpch/1
2734
targets: "datafusion:arrow,datafusion:parquet,datafusion:vortex,duckdb:parquet,duckdb:vortex,duckdb:duckdb"
35+
scale_factor: "--scale-factor ${{inputs.scale_factor}}"
2836
- id: clickbench-nvme
2937
binary_name: clickbench
3038
name: Clickbench on NVME
31-
local_dir: bench-vortex/data/clickbench_partitioned
3239
targets: "datafusion:parquet,datafusion:vortex,duckdb:parquet,duckdb:vortex,duckdb:duckdb"
40+
scale_factor: ""
3341
- id: tpch-s3
3442
binary_name: tpch
3543
name: TPC-H on S3
36-
local_dir: bench-vortex/data/tpch/1
37-
remote_storage: s3://vortex-bench-dev-eu/${{github.ref_name}}/tpch/1/
44+
local_dir: bench-vortex/data/tpch/${{inputs.scale_factor}}
45+
remote_storage: s3://vortex-bench-dev-eu/${{github.ref_name}}/tpch/${{inputs.scale_factor}}/
3846
targets: "datafusion:parquet,datafusion:vortex,duckdb:parquet"
47+
scale_factor: "--scale-factor ${{inputs.scale_factor}}"
48+
3949
runs-on:
4050
- runs-on=${{ github.run_id }}
41-
- family=c6id.8xlarge
51+
- family=${{ inputs.machine_type }}
4252
- image=ubuntu24-full-x64
4353
- spot=false
4454
- extras=s3-cache
@@ -77,9 +87,9 @@ jobs:
7787
RUST_BACKTRACE: full
7888
run: |
7989
# Generate data, running each query once to make sure they don't panic.
80-
target/release_debug/${{ matrix.binary_name }} --targets datafusion:parquet -i1 -d gh-json
81-
target/release_debug/${{ matrix.binary_name }} --targets datafusion:vortex -i1 -d gh-json
82-
target/release_debug/${{ matrix.binary_name }} --targets duckdb:vortex -i1 -d gh-json
90+
target/release_debug/${{ matrix.binary_name }} --targets datafusion:parquet -i1 -d gh-json ${{ matrix.scale_factor }}
91+
target/release_debug/${{ matrix.binary_name }} --targets datafusion:vortex -i1 -d gh-json ${{ matrix.scale_factor }}
92+
target/release_debug/${{ matrix.binary_name }} --targets duckdb:vortex -i1 -d gh-json ${{ matrix.scale_factor }}
8393
8494
- name: Setup AWS CLI
8595
uses: aws-actions/configure-aws-credentials@v4
@@ -117,6 +127,7 @@ jobs:
117127
-d gh-json \
118128
--targets ${{ matrix.targets }} \
119129
--export-spans \
130+
${{ matrix.scale_factor }} \
120131
-o results.json
121132
122133
- name: Run ${{ matrix.name }} benchmark (remote)
@@ -134,6 +145,7 @@ jobs:
134145
--use-remote-data-dir ${{ matrix.remote_storage }} \
135146
--targets ${{ matrix.targets }} \
136147
--export-spans \
148+
${{ matrix.scale_factor }} \
137149
-d gh-json \
138150
-o results.json
139151

‎benchmarks-website/code.js‎

Lines changed: 36 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,10 @@ window.initAndRender = (function () {
3838
"Random Access": new Map(),
3939
"Compression": new Map(),
4040
"Compression Size": new Map(),
41-
"TPC-H (NVME)": new Map(),
42-
"TPC-H (S3)": new Map(),
41+
"TPC-H (NVME) (SF=1)": new Map(),
42+
"TPC-H (S3) (SF=1)": new Map(),
43+
"TPC-H (NVME) (SF=100)": new Map(),
44+
"TPC-H (S3) (SF=100)": new Map(),
4345
"Clickbench": new Map(),
4446
};
4547

@@ -64,11 +66,28 @@ window.initAndRender = (function () {
6466

6567
let {name, unit, value, commit} = benchmark_result;
6668
let storage = benchmark_result.storage;
69+
let dataset = benchmark_result.dataset;
6770
let group = undefined;
6871
let group_id = undefined;
6972

7073

71-
if (name.startsWith("random-access/")) {
74+
if (dataset !== undefined) {
75+
if (dataset.tpch !== undefined) {
76+
let scale_factor = dataset.tpch.scale_factor;
77+
let nvme = storage === undefined || storage === "nvme";
78+
if (scale_factor === 1) {
79+
group_id = nvme ? "TPC-H (NVME) (SF=1)" : "TPC-H (S3) (SF=1)"
80+
} else if (scale_factor === 100) {
81+
group_id = nvme ? "TPC-H (NVME) (SF=100)" : "TPC-H (S3) (SF=100)"
82+
} else {
83+
console.warn("no scale factor found in benchmark")
84+
}
85+
} else if (dataset.clickbench !== undefined) {
86+
group_id = "Clickbench";
87+
} else {
88+
console.warn("unknown dataset please implement")
89+
}
90+
} else if (name.startsWith("random-access/")) {
7291
group_id = "Random Access";
7392
} else if (name.includes("compress time/")) {
7493
group_id = "Compression";
@@ -85,9 +104,9 @@ window.initAndRender = (function () {
85104
group_id = "Compression Size";
86105
} else if (name.startsWith("tpch_q")) {
87106
if (storage === undefined || storage === "nvme") {
88-
group_id = "TPC-H (NVME)";
107+
group_id = "TPC-H (NVME) (SF=1)";
89108
} else {
90-
group_id = "TPC-H (S3)";
109+
group_id = "TPC-H (S3) (SF=1)";
91110
}
92111
} else if (name.startsWith("clickbench")) {
93112
group_id = "Clickbench";
@@ -341,8 +360,10 @@ window.initAndRender = (function () {
341360
setElem.appendChild(graphsElem);
342361

343362
if (keptCharts === undefined) {
344-
for (const [benchName, benches] of benchSet.entries()) {
345-
charts.push(renderChart(graphsElem, benchName, benches, hiddenDatasets, removedDatasets, renamedDatasets))
363+
if (benchSet !== undefined) {
364+
for (const [benchName, benches] of benchSet.entries()) {
365+
charts.push(renderChart(graphsElem, benchName, benches, hiddenDatasets, removedDatasets, renamedDatasets))
366+
}
346367
}
347368
} else {
348369
for (const benchName of keptCharts) {
@@ -378,30 +399,30 @@ window.initAndRender = (function () {
378399

379400
async function fetchAndDecompressGzip(url) {
380401
const response = await fetch(url);
381-
402+
382403
const decompressedStream = response.body
383404
.pipeThrough(new DecompressionStream('gzip'));
384-
405+
385406
const reader = decompressedStream.getReader();
386407
const decoder = new TextDecoder();
387408
let result = '';
388-
409+
389410
while (true) {
390-
const { done, value } = await reader.read();
411+
const {done, value} = await reader.read();
391412
if (done) break;
392-
result += decoder.decode(value, { stream: true });
413+
result += decoder.decode(value, {stream: true});
393414
}
394-
415+
395416
result += decoder.decode();
396-
417+
397418
return result;
398419
}
399420

400421
function initAndRender(keptGroups) {
401422
let data = fetchAndDecompressGzip('https://vortex-benchmark-results-database.s3.amazonaws.com/data.json.gz')
402423
.then(parse_jsonl)
403424
.catch(error => console.error('unable to load data.json.gz:', error));
404-
425+
405426
let commit_metadata = fetch('https://vortex-benchmark-results-database.s3.amazonaws.com/commits.json')
406427
.then(response => response.text())
407428
.then(parse_jsonl)

‎benchmarks-website/index.html‎

Lines changed: 23 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -45,15 +45,11 @@
4545
],
4646
"hiddenDatasets": undefined,
4747
}],
48-
["TPC-H (NVME)", {
48+
["TPC-H (NVME) (SF=1)", {
4949
"keptCharts": undefined,
5050
"hiddenDatasets": undefined,
5151
"removedDatasets": new Set([]),
5252
"renamedDatasets": {
53-
"vortex-file-compressed": "datafusion:vortex",
54-
"parquet": "datafusion:parquet",
55-
"arrow": "datafusion:arrow",
56-
5753
"DataFusion:vortex-file-compressed": "datafusion:vortex",
5854
"DataFusion:parquet": "datafusion:parquet",
5955
"DataFusion:arrow": "datafusion:arrow",
@@ -63,14 +59,10 @@
6359
"DuckDB:duckdb": "duckdb:duckdb",
6460
},
6561
}],
66-
["TPC-H (S3)", {
62+
["TPC-H (S3) (SF=1)", {
6763
"keptCharts": undefined,
6864
"hiddenDatasets": undefined,
6965
"renamedDatasets": {
70-
"vortex-file-compressed": "datafusion:vortex",
71-
"parquet": "datafusion:parquet",
72-
"arrow": "datafusion:arrow",
73-
7466
"DataFusion:vortex-file-compressed": "datafusion:vortex",
7567
"DataFusion:parquet": "datafusion:parquet",
7668
"DataFusion:arrow": "datafusion:arrow",
@@ -82,10 +74,17 @@
8274
}],
8375
["Clickbench", {
8476
"renamedDatasets": {
85-
"vortex-file-compressed": "datafusion:vortex",
86-
"parquet": "datafusion:parquet",
87-
"arrow": "datafusion:arrow",
77+
"DataFusion:vortex-file-compressed": "datafusion:vortex",
78+
"DataFusion:parquet": "datafusion:parquet",
79+
"DataFusion:arrow": "datafusion:arrow",
8880

81+
"DuckDB:vortex-file-compressed": "duckdb:vortex",
82+
"DuckDB:parquet": "duckdb:parquet",
83+
"DuckDB:duckdb": "duckdb:duckdb",
84+
}
85+
}],
86+
["TPCH (NVME) (SF=100)", {
87+
"renamedDatasets": {
8988
"DataFusion:vortex-file-compressed": "datafusion:vortex",
9089
"DataFusion:parquet": "datafusion:parquet",
9190
"DataFusion:arrow": "datafusion:arrow",
@@ -95,6 +94,17 @@
9594
"DuckDB:duckdb": "duckdb:duckdb",
9695
}
9796
}],
97+
["TPC-H (S3) (SF=100)", {
98+
"renamedDatasets": {
99+
"DataFusion:vortex-file-compressed": "datafusion:vortex",
100+
"DataFusion:parquet": "datafusion:parquet",
101+
"DataFusion:arrow": "datafusion:arrow",
102+
103+
"DuckDB:vortex-file-compressed": "duckdb:vortex",
104+
"DuckDB:parquet": "duckdb:parquet",
105+
"DuckDB:duckdb": "duckdb:duckdb",
106+
},
107+
}]
98108
]);
99109
</script>
100110
</head>

0 commit comments

Comments
 (0)