-
Notifications
You must be signed in to change notification settings - Fork 9
Open
Description
Using compaction as shown below, and getting No files to calculate_task_parallelism
.
Is it valid error when nothing to compact?
Version: rev = "05d81842810f320175f6721ff21fab34385d7bae"
` let target_file_size = 256 * 1024 * 1024_u64;
let small_file_threshold = (target_file_size * 7) / 10;
let base_config = CompactionBaseConfigBuilder::default()
.target_file_size(target_file_size)
.build()
.unwrap();
let planning_config = CompactionPlanningConfigBuilder::default()
.base(base_config.clone())
.max_parallelism(1)
.small_file_threshold(small_file_threshold)
.max_task_total_size(1 * 1024 * 1024 * 1024)
.build()
.unwrap();
let execution_config = CompactionExecutionConfigBuilder::default()
.base(base_config)
.data_file_prefix("compacted".to_string())
.enable_normalized_column_identifiers(false)
.enable_dynamic_size_estimation(true)
.size_estimation_smoothing_factor(0.3)
.max_concurrent_closes(1)
.max_record_batch_rows(5000000)
.build()
.unwrap();
let compaction_config = CompactionConfigBuilder::default()
.execution(execution_config)
.planning(planning_config)
.build()
.unwrap();
// Run the compaction system
let compaction = CompactionBuilder::new(
catalog.clone(),
table_ident,
CompactionType::MergeSmallDataFiles,
)
.with_config(Arc::new(compaction_config))
.with_catalog_name("rest_catalog".to_string())
.build();
let result = compaction.compact().await.map_err(|err| {
tracing::error!("Error while executing the Compaction process for table={table_name} in warehouse={warehouse}: {err:?}");
TMError::Compaction(CompactionError::GeneralFailure {
table: table_name.clone(),
warehouse: warehouse.clone(),
message: err.to_string(),
source: Arc::new(err),
bt: bt!(),
})
})?;`
Metadata
Metadata
Assignees
Labels
No labels