Skip to content

Commit 5ef6a3b

Browse files
committed
parameterize oracle in integration test
1 parent 5aa61a5 commit 5ef6a3b

6 files changed

Lines changed: 301 additions & 204 deletions

File tree

datafusion-fuzzer.toml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,3 +30,8 @@ max_row_count = 100
3030
max_expr_level = 3
3131
max_table_count = 3
3232
max_insert_per_table = 20
33+
34+
# Supported oracles: NoCrash, NestedQueries.
35+
# Randomly select one oracle from the configured set for each query.
36+
oracles = ["NoCrash"]
37+
# oracles = ["NoCrash", "NestedQueries"]

src/cli/runner.rs

Lines changed: 27 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ use crate::common::{InclusionConfig, LogicalTable, Result};
1313
use crate::datasource_generator::dataset_generator::DatasetGenerator;
1414
use crate::fuzz_context::{GlobalContext, ctx_observability::display_all_tables};
1515
use crate::fuzz_runner::{record_query_with_time, update_stat_for_round_completion};
16-
use crate::oracle::{NoCrashOracle, Oracle, QueryContext, QueryExecutionResult};
16+
use crate::oracle::{Oracle, QueryContext, QueryExecutionResult};
1717
use crate::query_generator::stmt_select_def::SelectStatementBuilder;
1818

1919
use super::error_whitelist;
@@ -214,22 +214,12 @@ async fn execute_oracle_test(
214214
seed: u64,
215215
ctx: &Arc<GlobalContext>,
216216
) -> Result<bool> {
217-
// Create a deterministic RNG instance for this test
218-
let mut rng = StdRng::seed_from_u64(seed);
219-
220-
// === Select a random oracle ===
221-
// TODO: disabled views since joining too many table is slow
222-
let available_oracles: Vec<Box<dyn Oracle + Send>> = vec![
223-
Box::new(NoCrashOracle::new(seed, Arc::clone(ctx))),
224-
// Box::new(NestedQueriesOracle::new(seed, Arc::clone(ctx))),
225-
];
226-
let oracle_index = rng.random_range(0..available_oracles.len());
227-
let mut selected_oracle = available_oracles.into_iter().nth(oracle_index).unwrap();
217+
let mut randomly_selected_oracle = select_random_configured_oracle(seed, ctx);
228218

229-
info!("Selected oracle: {}", selected_oracle);
219+
info!("Selected oracle: {}", randomly_selected_oracle);
230220

231221
// === Generate query group ===
232-
let query_group = match selected_oracle.generate_query_group() {
222+
let query_group = match randomly_selected_oracle.generate_query_group() {
233223
Ok(group) => group,
234224
Err(e) => {
235225
let err_msg = format!("Failed to generate query group: {}", e);
@@ -250,7 +240,7 @@ async fn execute_oracle_test(
250240
round,
251241
query_index,
252242
seed,
253-
selected_oracle.name(),
243+
randomly_selected_oracle.name(),
254244
&query_group,
255245
)?;
256246

@@ -269,7 +259,7 @@ async fn execute_oracle_test(
269259
}
270260

271261
// === Validate execution results ===
272-
match selected_oracle
262+
match randomly_selected_oracle
273263
.validate_consistency(&execution_results)
274264
.await
275265
{
@@ -281,14 +271,31 @@ async fn execute_oracle_test(
281271
error!("Oracle test failed: {}", e);
282272

283273
// Log error report if available
284-
if let Ok(error_report) = selected_oracle.create_error_report(&execution_results) {
274+
if let Ok(error_report) =
275+
randomly_selected_oracle.create_error_report(&execution_results)
276+
{
285277
error!("Error Report:\n{}", error_report);
286278
}
287279
Ok(false)
288280
}
289281
}
290282
}
291283

284+
fn select_random_configured_oracle(seed: u64, ctx: &Arc<GlobalContext>) -> Box<dyn Oracle + Send> {
285+
// Randomly pick one oracle for this query; the configured oracle set bounds the choice.
286+
let available_oracles: Vec<Box<dyn Oracle + Send>> = ctx
287+
.runner_config
288+
.oracles
289+
.iter()
290+
.copied()
291+
.map(|oracle| oracle.build(seed, Arc::clone(ctx)))
292+
.collect();
293+
294+
let mut rng = StdRng::seed_from_u64(seed);
295+
let oracle_index = rng.random_range(0..available_oracles.len());
296+
available_oracles.into_iter().nth(oracle_index).unwrap()
297+
}
298+
292299
fn append_query_log(
293300
ctx: &Arc<GlobalContext>,
294301
round: u32,
@@ -475,6 +482,7 @@ mod tests {
475482
max_expr_level: 2,
476483
max_table_count: 3,
477484
max_insert_per_table: 20,
485+
oracles: vec![crate::oracle::ConfiguredOracle::NoCrash],
478486
};
479487

480488
// Collect results from multiple runs
@@ -593,6 +601,7 @@ mod tests {
593601
max_expr_level: 2,
594602
max_table_count: 3,
595603
max_insert_per_table: 20,
604+
oracles: vec![crate::oracle::ConfiguredOracle::NoCrash],
596605
};
597606

598607
let mut results_by_seed = Vec::new();
@@ -691,8 +700,7 @@ mod tests {
691700
for i in 0..ctx.runner_config.queries_per_round {
692701
let query_seed = query_base_seed.wrapping_add(i as u64);
693702

694-
// Generate a query using the same logic as execute_oracle_test
695-
let mut oracle = NoCrashOracle::new(query_seed, Arc::clone(&ctx));
703+
let mut oracle = select_random_configured_oracle(query_seed, &ctx);
696704
if let Ok(query_group) = oracle.generate_query_group() {
697705
if let Some(query_context) = query_group.first() {
698706
captured_queries

src/fuzz_context/mod.rs

Lines changed: 4 additions & 171 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,19 @@
11
pub mod ctx_observability;
2+
mod runner_config;
23

3-
use std::fs;
4-
use std::path::{Path, PathBuf};
54
use std::sync::{
65
Arc, Mutex, RwLock,
76
atomic::{AtomicU32, Ordering},
87
};
98

109
use datafusion::{common::HashMap, prelude::SessionContext};
11-
use serde::{Deserialize, Serialize};
1210

11+
use crate::common::LogicalTable;
1312
use crate::common::value_generator::ValueGenerationConfig;
14-
use crate::common::{LogicalTable, Result, fuzzer_err};
1513
use crate::fuzz_runner::FuzzerStats;
1614

15+
pub use runner_config::RunnerConfig;
16+
1717
/// Create a default DataFusion SessionContext with standard configuration
1818
/// This ensures consistency between initial creation and reset operations
1919
fn default_df_session_context() -> Arc<SessionContext> {
@@ -53,8 +53,6 @@ impl GlobalContext {
5353
/// Reset the DataFusion context to drop all registered tables
5454
/// This creates a fresh SessionContext and clears all table registrations
5555
pub fn reset_datafusion_context(&self) {
56-
use std::sync::atomic::Ordering;
57-
5856
// Create a new SessionContext to completely reset the DataFusion state
5957
let new_session_context = default_df_session_context();
6058

@@ -77,171 +75,6 @@ impl GlobalContext {
7775
}
7876
}
7977

80-
/// Unified configuration for the DataFusion fuzzer.
81-
///
82-
/// This configuration controls both:
83-
/// 1. The overall fuzzing process (rounds, queries, timeout)
84-
/// 2. The table and query generation parameters
85-
/// 3. UI and display parameters
86-
#[derive(Debug, Serialize, Deserialize, Clone)]
87-
pub struct RunnerConfig {
88-
// General fuzzing parameters
89-
pub seed: u64,
90-
pub rounds: u32,
91-
pub queries_per_round: u32,
92-
pub timeout_seconds: u64,
93-
pub log_path: Option<PathBuf>,
94-
95-
// UI and display parameters
96-
pub display_logs: bool,
97-
pub enable_tui: bool,
98-
pub sample_interval_secs: u64,
99-
100-
// Table and query generation parameters
101-
pub max_column_count: u64,
102-
pub max_row_count: u64,
103-
pub max_expr_level: u32,
104-
pub max_table_count: u32,
105-
pub max_insert_per_table: u32,
106-
}
107-
108-
impl RunnerConfig {
109-
pub fn new() -> Self {
110-
Self::default()
111-
}
112-
113-
pub fn with_seed(mut self, seed: u64) -> Self {
114-
self.seed = seed;
115-
self
116-
}
117-
118-
pub fn with_rounds(mut self, rounds: u32) -> Self {
119-
self.rounds = rounds;
120-
self
121-
}
122-
123-
pub fn with_queries_per_round(mut self, queries_per_round: u32) -> Self {
124-
self.queries_per_round = queries_per_round;
125-
self
126-
}
127-
128-
pub fn with_timeout_seconds(mut self, timeout_seconds: u64) -> Self {
129-
self.timeout_seconds = timeout_seconds;
130-
self
131-
}
132-
133-
pub fn with_log_path(mut self, log_path: Option<PathBuf>) -> Self {
134-
self.log_path = log_path;
135-
self
136-
}
137-
138-
pub fn with_display_logs(mut self, display_logs: bool) -> Self {
139-
self.display_logs = display_logs;
140-
self
141-
}
142-
143-
pub fn with_enable_tui(mut self, enable_tui: bool) -> Self {
144-
self.enable_tui = enable_tui;
145-
self
146-
}
147-
148-
pub fn with_sample_interval_secs(mut self, sample_interval_secs: u64) -> Self {
149-
self.sample_interval_secs = sample_interval_secs;
150-
self
151-
}
152-
153-
pub fn with_max_column_count(mut self, max_column_count: u64) -> Self {
154-
self.max_column_count = max_column_count;
155-
self
156-
}
157-
158-
pub fn with_max_row_count(mut self, max_row_count: u64) -> Self {
159-
self.max_row_count = max_row_count;
160-
self
161-
}
162-
163-
pub fn with_max_expr_level(mut self, max_expr_level: u32) -> Self {
164-
self.max_expr_level = max_expr_level;
165-
self
166-
}
167-
168-
pub fn with_max_table_count(mut self, max_table_count: u32) -> Self {
169-
self.max_table_count = max_table_count;
170-
self
171-
}
172-
173-
pub fn with_max_insert_per_table(mut self, max_insert_per_table: u32) -> Self {
174-
self.max_insert_per_table = max_insert_per_table;
175-
self
176-
}
177-
178-
pub fn from_file(path: &Path) -> Result<Self> {
179-
let content = fs::read_to_string(path)
180-
.map_err(|e| fuzzer_err(&format!("Failed to read config file: {}", e)))?;
181-
182-
let config: Self = toml::from_str(&content)
183-
.map_err(|e| fuzzer_err(&format!("Failed to parse config file: {}", e)))?;
184-
185-
Ok(config)
186-
}
187-
188-
pub fn from_cli(cli: &crate::cli::Cli) -> Result<Self> {
189-
// Start with default or config file if provided
190-
let mut config = if let Some(config_path) = &cli.config {
191-
Self::from_file(config_path)?
192-
} else {
193-
Self::default()
194-
};
195-
196-
// Override with CLI arguments if provided
197-
if cli.seed != 42 {
198-
config.seed = cli.seed;
199-
}
200-
201-
if let Some(rounds) = cli.rounds {
202-
config.rounds = rounds;
203-
}
204-
205-
if let Some(queries) = cli.queries_per_round {
206-
config.queries_per_round = queries;
207-
}
208-
209-
if let Some(timeout) = cli.timeout {
210-
config.timeout_seconds = timeout;
211-
}
212-
213-
if let Some(log_path) = &cli.log_path {
214-
config.log_path = Some(log_path.clone());
215-
}
216-
217-
// Set display_logs from CLI argument
218-
config.display_logs = cli.display_logs;
219-
220-
// Set enable_tui from CLI argument
221-
config.enable_tui = cli.enable_tui;
222-
223-
Ok(config)
224-
}
225-
226-
pub fn default() -> Self {
227-
Self {
228-
seed: 42,
229-
rounds: 3,
230-
queries_per_round: 10,
231-
timeout_seconds: 2,
232-
log_path: Some(PathBuf::from("logs")),
233-
display_logs: false,
234-
enable_tui: true,
235-
sample_interval_secs: 5,
236-
max_column_count: 5,
237-
max_row_count: 100,
238-
max_expr_level: 3,
239-
max_table_count: 3,
240-
max_insert_per_table: 20,
241-
}
242-
}
243-
}
244-
24578
pub struct RuntimeContext {
24679
pub df_ctx: Arc<RwLock<Arc<SessionContext>>>,
24780
pub registered_tables: Arc<RwLock<HashMap<String, Arc<LogicalTable>>>>,

0 commit comments

Comments
 (0)