Skip to content

Commit 087aaf0

Browse files
ilaila
committed
benchmark: archive paper result inputs
1 parent f09d76c commit 087aaf0

20 files changed

Lines changed: 1387 additions & 777 deletions

‎.gitignore‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,10 @@ test/python/__pycache__/
1818
benchmark/**/*.png
1919
benchmark/**/*.log
2020
benchmark/**/__pycache__/
21-
benchmark/results/
21+
benchmark/results/*
22+
!benchmark/results/paper/
23+
!benchmark/results/paper/*.csv
24+
!benchmark/results/paper/README.md
2225
benchmark/dp/tmp/
2326
benchmark/tpch/dp_elastic_*.csv
2427
benchmark/tpch/dp_sass_*.csv

‎CMakeLists.txt‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -110,10 +110,6 @@ if(NOT EMSCRIPTEN AND NOT WIN32)
110110
add_executable(dp_benchmark_runner benchmark/dp/dp_benchmark_runner.cpp)
111111
target_link_libraries(dp_benchmark_runner ${PRIVACY_LINK_LIBS})
112112

113-
# Target-metric postprocessor for selected DP benchmark rows
114-
add_executable(dp_target_metric_runner benchmark/dp/dp_target_metric_runner.cpp)
115-
target_link_libraries(dp_target_metric_runner ${PRIVACY_LINK_LIBS})
116-
117113
# IMDb/JOB benchmark runner
118114
add_executable(imdb_benchmark benchmark/imdb/imdb_benchmark.cpp)
119115
target_link_libraries(imdb_benchmark ${PRIVACY_LINK_LIBS})

‎benchmark/dp/dp_target_metric_runner.cpp‎

Lines changed: 0 additions & 770 deletions
This file was deleted.

‎benchmark/results/paper/README.md‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
# Paper Result Data
2+
3+
This directory contains the CSV inputs retained for the submitted paper figures. The files are immutable reference
4+
artifacts; new runs use `benchmark/results/utility/` and `benchmark/results/sanity/`.
5+
6+
| File | Contents |
7+
|---|---|
8+
| `as_clickbench_m64_m512.csv` | ClickBench DuckDB and SIMD-AS runtimes |
9+
| `as_tpch_m64.csv` | TPC-H DuckDB, SIMD-AS, naive-AS, and simple-hash runtimes |
10+
| `as_tpch_m_sweep.csv` | Corrected TPC-H DuckDB and variable-`m` SIMD-AS run |
11+
| `as_tpch_m_figure.csv` | TPC-H variable-`m` figure summary |
12+
| `dp_tpch_utility_raw.csv` | Main TPC-H DP utility sweep |
13+
| `dp_tpch_m_figure.csv` | SAA target-error variable-`m` figure summary |
14+
| `dp_tpch_runtime_raw.csv` | TPC-H DP runtime measurements |
15+
| `dp_tpch_runtime_figure.csv` | TPC-H DP runtime figure summary |
16+
| `q01_avg_columns_raw.csv` | Three-run Q01 AVG-column measurements |
17+
| `q01_avg_groups_raw.csv` | Three-run Q01 AVG grouping measurements |
18+
| `q01_avg_figure.csv` | Q01 AVG figure values |
19+
| `q01_stability_avg_summary_raw.csv` | AVG lane-stability measurements |
20+
| `q01_stability_sum_count_summary_raw.csv` | Rescaled SUM/COUNT lane-stability measurements |
21+
| `q01_stability_figure.csv` | Q01 lane-stability figure values |
22+
23+
The JSON configurations in `benchmark/configs/utility/` define the corresponding experiments. See
24+
[`repro.md`](../../../repro.md) for commands, bounds, metrics, and machine details.

‎benchmark/results/paper/as_clickbench_m64_m512.csv‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
query,mode,m,median_ms
2+
q01,baseline,0,0.361028
3+
q01,SIMD-AS,64,39.750133
4+
q01,SIMD-AS,512,14.950117
5+
q02,baseline,0,2.203463
6+
q02,SIMD-AS,64,2.927227
7+
q02,SIMD-AS,512,2.678774
8+
q03,baseline,0,13.27081
9+
q03,SIMD-AS,64,498.832857
10+
q03,SIMD-AS,512,71.64823
11+
q04,baseline,0,19.468944
12+
q04,SIMD-AS,64,258.898227
13+
q04,SIMD-AS,512,50.788924
14+
q05,baseline,0,128.684616
15+
q05,SIMD-AS,64,181.844913
16+
q05,SIMD-AS,512,178.673178
17+
q06,baseline,0,203.305313
18+
q06,SIMD-AS,64,228.317982
19+
q06,SIMD-AS,512,225.160102
20+
q07,baseline,0,3.17155
21+
q07,SIMD-AS,64,-1
22+
q07,SIMD-AS,512,-1
23+
q08,baseline,0,3.440415
24+
q08,SIMD-AS,64,4.108121
25+
q08,SIMD-AS,512,4.652447
26+
q09,baseline,0,170.865637
27+
q09,SIMD-AS,64,240.155728
28+
q09,SIMD-AS,512,242.888176
29+
q10,baseline,0,252.1943
30+
q10,SIMD-AS,64,830.85007
31+
q10,SIMD-AS,512,438.62556
32+
q11,baseline,0,57.187048
33+
q11,SIMD-AS,64,59.895437
34+
q11,SIMD-AS,512,58.981159
35+
q12,baseline,0,62.281941
36+
q12,SIMD-AS,64,66.227829
37+
q12,SIMD-AS,512,66.036838
38+
q13,baseline,0,181.298327
39+
q13,SIMD-AS,64,326.19136
40+
q13,SIMD-AS,512,-1
41+
q14,baseline,0,324.790997
42+
q14,SIMD-AS,64,482.825666
43+
q14,SIMD-AS,512,-1
44+
q15,baseline,0,207.473684
45+
q15,SIMD-AS,64,355.000114
46+
q15,SIMD-AS,512,-1
47+
q16,baseline,0,283.260678
48+
q16,SIMD-AS,64,729.048688
49+
q16,SIMD-AS,512,-1
50+
q17,baseline,0,395.880051
51+
q17,SIMD-AS,64,1052.253422
52+
q17,SIMD-AS,512,-1
53+
q18,baseline,0,319.031375
54+
q18,SIMD-AS,64,394.821666
55+
q18,SIMD-AS,512,-1
56+
q19,baseline,0,682.434456
57+
q19,SIMD-AS,64,2041.337462
58+
q19,SIMD-AS,512,-1
59+
q21,baseline,0,285.758731
60+
q21,SIMD-AS,64,286.885602
61+
q21,SIMD-AS,512,288.587516
62+
q22,baseline,0,250.814098
63+
q22,SIMD-AS,64,251.617215
64+
q22,SIMD-AS,512,249.548714
65+
q23,baseline,0,286.873769
66+
q23,SIMD-AS,64,535.363053
67+
q23,SIMD-AS,512,550.932387
68+
q28,baseline,0,192.860316
69+
q28,SIMD-AS,64,449.867547
70+
q28,SIMD-AS,512,224.262263
71+
q29,baseline,0,4081.59554
72+
q29,SIMD-AS,64,4548.97892
73+
q29,SIMD-AS,512,-1
74+
q30,baseline,0,20.902081
75+
q30,SIMD-AS,64,18855.907177
76+
q30,SIMD-AS,512,1756.192284
77+
q31,baseline,0,175.70805
78+
q31,SIMD-AS,64,913.130187
79+
q31,SIMD-AS,512,-1
80+
q32,baseline,0,213.095122
81+
q32,SIMD-AS,64,1561.108183
82+
q32,SIMD-AS,512,-1
83+
q33,baseline,0,796.401581
84+
q33,SIMD-AS,64,-1
85+
q33,SIMD-AS,512,-1
86+
q34,baseline,0,721.558652
87+
q34,SIMD-AS,64,1292.679509
88+
q34,SIMD-AS,512,-1
89+
q35,baseline,0,783.658863
90+
q35,SIMD-AS,64,1305.05343
91+
q35,SIMD-AS,512,-1
92+
q36,baseline,0,208.973325
93+
q36,SIMD-AS,64,637.327703
94+
q36,SIMD-AS,512,-1
95+
q37,baseline,0,17.947074
96+
q37,SIMD-AS,64,25.557312
97+
q37,SIMD-AS,512,98.897448
98+
q38,baseline,0,5.352259
99+
q38,SIMD-AS,64,7.399526
100+
q38,SIMD-AS,512,18.721606
101+
q39,baseline,0,7.0175
102+
q39,SIMD-AS,64,6.855789
103+
q39,SIMD-AS,512,10.446483
104+
q40,baseline,0,32.79514
105+
q40,SIMD-AS,64,43.817824
106+
q40,SIMD-AS,512,153.841202
107+
q41,baseline,0,8.717204
108+
q41,SIMD-AS,64,7.585793
109+
q41,SIMD-AS,512,18.082267
110+
q42,baseline,0,3.135017
111+
q42,SIMD-AS,64,3.82609
112+
q42,SIMD-AS,512,7.835368
113+
q43,baseline,0,7.051776
114+
q43,SIMD-AS,64,5.81753
115+
q43,SIMD-AS,512,6.756981

‎benchmark/results/paper/as_tpch_m64.csv‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
query,mode,median_ms
2+
q01,baseline,296.834543
3+
q01,SIMD AS,3159.099322
4+
q01,simple hash AS,1913.138977
5+
q01,Naive-AS,64776.56672
6+
q03,baseline,219.318654
7+
q03,SIMD AS,229.836629
8+
q03,simple hash AS,240.189163
9+
q03,Naive-AS,21575.280576
10+
q04,baseline,151.244998
11+
q04,SIMD AS,158.108942
12+
q04,simple hash AS,149.86101
13+
q04,Naive-AS,13764.076288
14+
q05,baseline,236.603723
15+
q05,SIMD AS,239.152468
16+
q05,simple hash AS,237.924573
17+
q05,Naive-AS,13733.765504
18+
q06,baseline,65.524952
19+
q06,SIMD AS,171.659665
20+
q06,simple hash AS,86.330723
21+
q06,Naive-AS,13586.754368
22+
q07,baseline,162.70839
23+
q07,SIMD AS,158.811835
24+
q07,simple hash AS,158.513194
25+
q07,Naive-AS,13621.981952
26+
q08,baseline,172.957798
27+
q08,SIMD AS,177.744985
28+
q08,simple hash AS,175.655583
29+
q08,Naive-AS,11605.693824
30+
q09,baseline,699.820454
31+
q09,SIMD AS,751.097755
32+
q09,simple hash AS,742.239503
33+
q09,Naive-AS,47822.03008
34+
q10,baseline,320.261267
35+
q10,SIMD AS,350.304301
36+
q10,simple hash AS,352.491847
37+
q10,Naive-AS,17066.548096
38+
q12,baseline,176.826018
39+
q12,SIMD AS,181.88052
40+
q12,simple hash AS,180.132466
41+
q12,Naive-AS,29788.241536
42+
q13,baseline,608.779744
43+
q13,SIMD AS,614.533312
44+
q13,simple hash AS,606.487534
45+
q13,Naive-AS,45865.633088
46+
q14,baseline,145.005063
47+
q14,SIMD AS,407.373506
48+
q14,simple hash AS,150.399822
49+
q14,Naive-AS,35388.05024
50+
q15,baseline,95.464373
51+
q15,SIMD AS,745.440545
52+
q15,simple hash AS,372.463032
53+
q15,Naive-AS,28766.5536
54+
q17,baseline,212.895415
55+
q17,SIMD AS,542.244258
56+
q17,simple hash AS,248.851791
57+
q17,Naive-AS,18753.867328
58+
q18,baseline,504.637785
59+
q18,SIMD AS,486.235654
60+
q18,simple hash AS,492.644723
61+
q18,Naive-AS,30373.645888
62+
q19,baseline,244.375757
63+
q19,SIMD AS,277.62932
64+
q19,simple hash AS,262.567906
65+
q19,Naive-AS,19749.410688
66+
q20,baseline,163.28468
67+
q20,SIMD AS,235.528722
68+
q20,simple hash AS,220.476851
69+
q20,Naive-AS,15482.924224
70+
q21,baseline,587.262664
71+
q21,SIMD AS,722.330429
72+
q21,simple hash AS,565.228428
73+
q21,Naive-AS,28977.435136
74+
q22,baseline,121.373803
75+
q22,SIMD AS,134.590089
76+
q22,simple hash AS,126.451375
77+
q22,Naive-AS,6068.99776

‎benchmark/results/paper/as_tpch_m_figure.csv‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
m_label,queries,median_slowdown,mean_slowdown,within_2x,within_3x,max_slowdown,max_query
2+
m=128,19,1.1525324922669404,2.7970045060709956,13,16,19.040411272136456,Q1
3+
m=256,19,1.2774644967712763,3.982504459504176,13,15,35.28326435162423,Q1
4+
m=512,19,1.4545346905593746,6.167046364163178,12,14,65.009533611828,Q1

‎benchmark/results/paper/as_tpch_m_sweep.csv‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
query,mode,m,median_ms
2+
q01,baseline,0,295.968312
3+
q01,SIMD AS,64,3105.227692
4+
q01,SIMD AS,128,5635.358384
5+
q01,SIMD AS,256,10442.728192
6+
q01,SIMD AS,512,19240.761927
7+
q03,baseline,0,207.324345
8+
q03,SIMD AS,64,215.715244
9+
q03,SIMD AS,128,227.992587
10+
q03,SIMD AS,256,253.70596
11+
q03,SIMD AS,512,301.560452
12+
q04,baseline,0,144.220085
13+
q04,SIMD AS,64,149.496211
14+
q04,SIMD AS,128,166.218334
15+
q04,SIMD AS,256,186.740221
16+
q04,SIMD AS,512,217.416507
17+
q05,baseline,0,216.912472
18+
q05,SIMD AS,64,218.346802
19+
q05,SIMD AS,128,225.745701
20+
q05,SIMD AS,256,228.438583
21+
q05,SIMD AS,512,218.67789
22+
q06,baseline,0,62.626039
23+
q06,SIMD AS,64,163.509116
24+
q06,SIMD AS,128,173.567755
25+
q06,SIMD AS,256,185.000168
26+
q06,SIMD AS,512,208.800031
27+
q07,baseline,0,155.137803
28+
q07,SIMD AS,64,160.523252
29+
q07,SIMD AS,128,158.907556
30+
q07,SIMD AS,256,153.180457
31+
q07,SIMD AS,512,157.078164
32+
q08,baseline,0,166.641774
33+
q08,SIMD AS,64,167.253051
34+
q08,SIMD AS,128,168.268221
35+
q08,SIMD AS,256,172.059086
36+
q08,SIMD AS,512,178.967823
37+
q09,baseline,0,686.054225
38+
q09,SIMD AS,64,721.781691
39+
q09,SIMD AS,128,747.255198
40+
q09,SIMD AS,256,784.162412
41+
q09,SIMD AS,512,866.774089
42+
q10,baseline,0,304.276456
43+
q10,SIMD AS,64,334.413126
44+
q10,SIMD AS,128,386.444783
45+
q10,SIMD AS,256,464.844489
46+
q10,SIMD AS,512,644.651732
47+
q12,baseline,0,173.46971
48+
q12,SIMD AS,64,175.025234
49+
q12,SIMD AS,128,178.873337
50+
q12,SIMD AS,256,186.96162
51+
q12,SIMD AS,512,198.118645
52+
q13,baseline,0,578.622349
53+
q13,SIMD AS,64,587.772151
54+
q13,SIMD AS,128,604.588652
55+
q13,SIMD AS,256,620.972592
56+
q13,SIMD AS,512,656.438781
57+
q14,baseline,0,124.478707
58+
q14,SIMD AS,64,394.410936
59+
q14,SIMD AS,128,401.343176
60+
q14,SIMD AS,256,415.732475
61+
q14,SIMD AS,512,449.525992
62+
q15,baseline,0,96.180306
63+
q15,SIMD AS,64,728.988695
64+
q15,SIMD AS,128,790.457463
65+
q15,SIMD AS,256,1131.315769
66+
q15,SIMD AS,512,1736.54136
67+
q17,baseline,0,211.146792
68+
q17,SIMD AS,64,532.053201
69+
q17,SIMD AS,128,547.562246
70+
q17,SIMD AS,256,579.03088
71+
q17,SIMD AS,512,626.632677
72+
q18,baseline,0,477.442289
73+
q18,SIMD AS,64,468.814389
74+
q18,SIMD AS,128,460.341325
75+
q18,SIMD AS,256,465.106129
76+
q18,SIMD AS,512,466.832628
77+
q19,baseline,0,247.256871
78+
q19,SIMD AS,64,271.183055
79+
q19,SIMD AS,128,272.651864
80+
q19,SIMD AS,256,271.412116
81+
q19,SIMD AS,512,274.738109
82+
q20,baseline,0,158.448386
83+
q20,SIMD AS,64,224.669123
84+
q20,SIMD AS,128,232.394539
85+
q20,SIMD AS,256,242.990823
86+
q20,SIMD AS,512,268.398972
87+
q21,baseline,0,540.989459
88+
q21,SIMD AS,64,681.570249
89+
q21,SIMD AS,128,696.036873
90+
q21,SIMD AS,256,691.094827
91+
q21,SIMD AS,512,686.223415
92+
q22,baseline,0,116.647105
93+
q22,SIMD AS,64,126.397216
94+
q22,SIMD AS,128,316.43477
95+
q22,SIMD AS,256,500.368596
96+
q22,SIMD AS,512,866.636819

‎benchmark/results/paper/dp_tpch_m_figure.csv‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
query,dp_sass_m,sampled_error,rows,m_label,plot_error
2+
Q01,64,1.2722798323966067,3,m=64,1.2722798323966067
3+
Q01,128,0.9846025474518959,3,m=128,0.9846025474518959
4+
Q01,256,1.0218994860204305,3,m=256,1.0218994860204305
5+
Q01,512,0.9074844527565946,3,m=512,0.9074844527565946
6+
Q05,64,0.3834434351285753,3,m=64,0.3834434351285753
7+
Q05,128,0.33026033230894936,3,m=128,0.33026033230894936
8+
Q05,256,0.278192031743807,3,m=256,0.278192031743807
9+
Q05,512,0.25665425246049567,3,m=512,0.25665425246049567
10+
Q06,64,0.0242438276819611,3,m=64,0.0242438276819611
11+
Q06,128,0.015582577024358845,3,m=128,0.015582577024358845
12+
Q06,256,0.012837078734432907,3,m=256,0.012837078734432907
13+
Q06,512,0.009239330117614847,3,m=512,0.009239330117614847
14+
Q14,64,0.17896399445555225,3,m=64,0.17896399445555225
15+
Q14,128,0.1075926257685558,3,m=128,0.1075926257685558
16+
Q14,256,0.073597831323695,3,m=256,0.073597831323695
17+
Q14,512,0.05198244265800038,3,m=512,0.05198244265800038
18+
Q19,64,0.6226431412483896,3,m=64,0.6226431412483896
19+
Q19,128,0.35252021565867736,3,m=128,0.35252021565867736
20+
Q19,256,0.2565477959639743,3,m=256,0.2565477959639743
21+
Q19,512,0.18687703428272637,3,m=512,0.18687703428272637

‎benchmark/results/paper/dp_tpch_runtime_figure.csv‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
query,mode,runs,median_time_ms,mean_time_ms,duckdb_median_time_ms,slowdown_vs_duckdb_median
2+
q01,DuckDB,3,347.473255,348.233168,347.473255,1.000000
3+
q01,DP-Bounded,3,3886.446746,3895.639838,347.473255,11.184880
4+
q01,DP-Elastic,3,2801.291009,2795.685967,347.473255,8.061890
5+
q01,DP-SAA m=64,3,3408.725301,3406.054951,347.473255,9.810036
6+
q01,DP-SAA m=512,3,3325.554877,3320.193291,347.473255,9.570679
7+
q05,DuckDB,3,312.678782,313.462673,312.678782,1.000000
8+
q05,DP-Bounded,3,549.720453,549.263122,312.678782,1.758100
9+
q05,DP-Elastic,3,922.595688,922.656156,312.678782,2.950618
10+
q05,DP-SAA m=64,3,539.200498,539.160582,312.678782,1.724455
11+
q05,DP-SAA m=512,3,545.554019,546.092941,312.678782,1.744775
12+
q06,DuckDB,3,117.135526,116.129771,117.135526,1.000000
13+
q06,DP-Bounded,3,330.931410,329.555395,117.135526,2.825201
14+
q06,DP-Elastic,3,615.210948,615.671902,117.135526,5.252129
15+
q06,DP-SAA m=64,3,325.605215,325.012254,117.135526,2.779731
16+
q06,DP-SAA m=512,3,330.994039,329.921927,117.135526,2.825736
17+
q14,DuckDB,3,200.094996,198.563932,200.094996,1.000000
18+
q14,DP-Bounded,3,613.603993,615.307059,200.094996,3.066563
19+
q14,DP-Elastic,3,719.494281,721.453032,200.094996,3.595763
20+
q14,DP-SAA m=64,3,650.182228,644.412135,200.094996,3.249368
21+
q14,DP-SAA m=512,3,619.540361,620.557095,200.094996,3.096231
22+
q19,DuckDB,3,311.180564,313.007652,311.180564,1.000000
23+
q19,DP-Bounded,3,576.705250,577.911865,311.180564,1.853282
24+
q19,DP-Elastic,3,907.825989,908.330049,311.180564,2.917361
25+
q19,DP-SAA m=64,3,574.027908,575.350554,311.180564,1.844678
26+
q19,DP-SAA m=512,3,581.488020,580.143241,311.180564,1.868651

0 commit comments

Comments
 (0)