benchmark: warm caches before timed queries

Author: ila

benchmark/clickbench/as_clickbench_benchmark.cpp

@@ -664,8 +664,15 @@ int RunASClickBenchBenchmark(const string &db_path, const string &queries_dir, c
 						}
 						query_sql = RewriteAsQueryForSampleAggregation(as_sql, as_m);
 					}
-					double as_median =
-					    RunMedian(con, query_sql, 5, "SIMD-AS " + entry.label + " m=" + std::to_string(as_m));
+					string warmup_error;
+					double as_median = -1;
+					if (!RunSql(con, query_sql, warmup_error)) {
+						Log("SIMD-AS " + entry.label + " m=" + std::to_string(as_m) + " warmup error: " + warmup_error);
+					} else {
+						Log("SIMD-AS " + entry.label + " m=" + std::to_string(as_m) + " warmup complete");
+						as_median =
+						    RunMedian(con, query_sql, 5, "SIMD-AS " + entry.label + " m=" + std::to_string(as_m));
+					}
 					csv << entry.label << ",SIMD-AS," << as_m << "," << FormatNumber(as_median) << "\n";
 				}
 				con.Query("SET priv_rewrite=true;");

benchmark/dp/dp_benchmark_runner.cpp

@@ -1905,8 +1905,8 @@ static DatasetRun CreateDatasetRun(const Config &config, const DatasetConfig &da
 	}
 	result.points = BuildRunPoints(dataset);
 	result.db_path = DefaultDbPath(dataset);
-	if (!(dataset.name == "tpch" || dataset.name == "sqlstorm_tpch") &&
-	    !dataset.allow_create && !FileExists(result.db_path)) {
+	if (!(dataset.name == "tpch" || dataset.name == "sqlstorm_tpch") && !dataset.allow_create &&
+	    !FileExists(result.db_path)) {
 		throw std::runtime_error("database does not exist for dataset " + dataset.name + ": " + result.db_path);
 	}
 
@@ -1975,6 +1975,14 @@ static void PrepareDatasetRun(const Config &config, DatasetRun &dataset_run,
 	PrepareDataset(*dataset_run.connection, dataset_run.config, false);
 }
 
+static void ExecuteBenchmarkQuery(Connection &con, const BenchmarkRow &row, const QuerySpec &query) {
+	if (row.point.mode == "dp_sass_bounded_ratio") {
+		ReadScalarDouble(con, BuildQ14BoundedSampledRatioSql(row.point.epsilon, row.point.sample_lanes));
+	} else {
+		MaterializeQuery(con, query.sql);
+	}
+}
+
 static void RunTimingPass(const Config &config, DatasetRun &dataset_run) {
 	auto &con = *dataset_run.connection;
 	vector<double> pu_counts(dataset_run.queries.size(), -1.0);
@@ -1997,12 +2005,11 @@ static void RunTimingPass(const Config &config, DatasetRun &dataset_run) {
 			RunStatement(con, "SET privacy_noise=true");
 			RunStatement(con, "SET privacy_seed=" + std::to_string(row.seed));
 
+			ExecuteBenchmarkQuery(con, row, query);
+			Log("warmup " + dataset_run.config.name + " " + row.point.mode + " " + query.name + " run " +
+			    std::to_string(row.run) + " complete");
 			auto start = std::chrono::steady_clock::now();
-			if (row.point.mode == "dp_sass_bounded_ratio") {
-				ReadScalarDouble(con, BuildQ14BoundedSampledRatioSql(row.point.epsilon, row.point.sample_lanes));
-			} else {
-				MaterializeQuery(con, query.sql);
-			}
+			ExecuteBenchmarkQuery(con, row, query);
 			auto end = std::chrono::steady_clock::now();
 			row.time_ms = std::chrono::duration<double, std::milli>(end - start).count();
 			row.runtime_success = true;

benchmark/tpch/as_tpch_benchmark.cpp

@@ -1048,6 +1048,16 @@ int RunASTPCHBenchmark(const string &db_path, const string &queries_dir, double
 								query_sql = RewriteAsQueryForSampleAggregation(as_sql, as_m);
 							}
 						}
+						con.Query("SET pac_m=" + std::to_string(as_m) + ";");
+						con.Query("SET priv_rewrite=false;");
+						auto r_warmup = con.Query(query_sql);
+						if (r_warmup && r_warmup->HasError()) {
+							Log("AS (" + mode_str + ") " + entry.label + " m=" + std::to_string(as_m) +
+							    " warmup error: " + r_warmup->GetError());
+							csv << entry.label << "," << mode_str << "," << as_m << ",-1\n";
+							continue;
+						}
+						Log("AS (" + mode_str + ") " + entry.label + " m=" + std::to_string(as_m) + " warmup complete");
 						// Run AS query 5 times, take median
 						vector<double> as_times_ms;
 						bool as_failed = false;
@@ -1113,6 +1123,14 @@ int RunASTPCHBenchmark(const string &db_path, const string &queries_dir, double
 							if (!setup_ok) {
 								csv << entry.label << ",Naive-AS,64,-1\n";
 							} else {
+								auto r_warmup = con.Query(execute_sql);
+								if (r_warmup && r_warmup->HasError()) {
+									Log("Naive-AS " + entry.label + " warmup error: " + r_warmup->GetError());
+									csv << entry.label << ",Naive-AS,64,-1\n";
+									con.Query("DEALLOCATE PREPARE run_query;");
+									continue;
+								}
+								Log("Naive-AS " + entry.label + " warmup complete");
 								// Time EXECUTE 5 times, take median
 								vector<double> naive_as_times_ms;
 								bool naive_as_failed = false;

repro.md

@@ -87,6 +87,8 @@ build/release/extension/privacy/as_tpch_benchmark \
   --config benchmark/configs/utility/as_tpch_sf30.json
 ```
 
+Each AS query/mode is executed once without timing before its five measured executions. The CSV reports their median.
+
 The variable-`m` configuration measures the separate aggregate path for `m=128,256,512`. It reuses the same database
 and skips the naive implementation.
 
@@ -111,6 +113,8 @@ build/release/extension/privacy/as_clickbench_benchmark \
   --config benchmark/configs/utility/as_clickbench_graviton.json
 ```
 
+Each AS query/mode is executed once without timing before its five measured executions. The CSV reports their median.
+
 Output: `benchmark/results/utility/as_clickbench_graviton.csv`.
 
 ### DP utility and runtime
@@ -134,6 +138,9 @@ build/release/extension/privacy/dp_benchmark_runner \
   --config benchmark/configs/utility/dp_tpch_sf30_utility.json
 ```
 
+The runner executes one untimed warmup immediately before every measured private-query execution. Utility and diagnostic
+queries run separately and are not included in `time_ms`.
+
 Output: `benchmark/results/utility/dp_tpch_sf30_utility.csv`.
 
 The runner deliberately uses two passes. It first times every private query and checkpoints the CSV after each