refactor: add config for datablock size

Author: Subsegment

config/config.toml

@@ -67,6 +67,9 @@ strict_write = false
 ## copyinto trigger flush size
 #copyinto_trigger_flush_size = "128M" # 134217728
 
+## The maximum size of a datablock in compaction. 
+max_datablock_size = 102400
+
 [wal]
 
 ## If true, write requets on disk before writing to memory.

config/config_8902.toml

@@ -40,6 +40,7 @@ max_concurrent_compaction = 4
 strict_write = false
 reserve_space = "0"
 copyinto_trigger_flush_size = "128M" # 134217728
+max_datablock_size = 102400
 
 [wal]
 enabled = true

config/config_8912.toml

@@ -40,6 +40,7 @@ max_concurrent_compaction = 4
 strict_write = false
 reserve_space = "0"
 copyinto_trigger_flush_size = "128M" # 134217728
+max_datablock_size = 102400
 
 [wal]
 enabled = true

config/config_8922.toml

@@ -39,6 +39,7 @@ max_compact_size = "2G"              # 2147483648
 max_concurrent_compaction = 4
 strict_write = false
 reserve_space = "0"
+max_datablock_size = 102400
 
 [wal]
 enabled = true

config/src/lib.rs

@@ -326,6 +326,9 @@ strict_write = false
 # The size of reserve space of the system.
 reserve_space = "10G"
 
+## The maximum size of a datablock in compaction.
+max_datablock_size = 102400
+
 [wal]
 
 ## If true, write requets on disk before writing to memory.

config/src/storage_config.rs

@@ -59,6 +59,9 @@ pub struct StorageConfig {
         default = "StorageConfig::default_copyinto_trigger_flush_size"
     )]
     pub copyinto_trigger_flush_size: u64,
+
+    #[serde(default = "StorageConfig::default_max_datablock_size")]
+    pub max_datablock_size: u64,
 }
 
 impl StorageConfig {
@@ -115,6 +118,10 @@ impl StorageConfig {
         128 * 1024 * 1024 // 128M
     }
 
+    fn default_max_datablock_size() -> u64 {
+        100 * 1024
+    }
+
     pub fn introspect(&mut self) {
         // Unit of storage.compact_trigger_cold_duration is seconds
         self.compact_trigger_cold_duration =
@@ -160,6 +167,11 @@ impl OverrideByEnv for StorageConfig {
             &mut self.copyinto_trigger_flush_size,
             "CNOSDB_COPYINTO_TRIGGER_FLUSH_SIZE",
         );
+        entry_override(&mut self.reserve_space, "CNOSDB_STORAGE_RESERVE_SPACE");
+        entry_override(
+            &mut self.max_datablock_size,
+            "CNOSDB_STORAGE_MAX_DATABLOCK_SIZE",
+        );
     }
 }
 
@@ -179,6 +191,7 @@ impl Default for StorageConfig {
             strict_write: Self::default_strict_write(),
             reserve_space: Self::default_reserve_space(),
             copyinto_trigger_flush_size: Self::default_copyinto_trigger_flush_size(),
+            max_datablock_size: Self::default_max_datablock_size(),
         }
     }
 }

tskv/src/compaction/compact.rs

@@ -12,7 +12,6 @@ use crate::compaction::CompactReq;
 use crate::context::GlobalContext;
 use crate::error::TskvResult;
 use crate::summary::{CompactMeta, VersionEdit};
-use crate::tseries_family::TseriesFamily;
 use crate::tsm::chunk::Chunk;
 use crate::tsm::column_group::ColumnGroup;
 use crate::tsm::data_block::DataBlock;
@@ -770,7 +769,7 @@ pub async fn run_compaction_job(
         tsm_readers.push(tsm_reader);
     }
 
-    let max_block_size = TseriesFamily::MAX_DATA_BLOCK_SIZE as usize;
+    let max_block_size = request.storage_opt.max_datablock_size as usize;
     let mut iter = CompactIterator::new(tsm_readers);
     let tsm_dir = request.storage_opt.tsm_dir(&request.database, tsf_id);
     let max_file_size = request.storage_opt.level_max_file_size(request.out_level);
@@ -1058,6 +1057,7 @@ pub mod test {
     pub(crate) fn create_options(base_dir: String) -> Arc<Options> {
         let mut config = config::get_config_for_test();
         config.storage.path = base_dir;
+        config.storage.max_datablock_size = 1000;
         let opt = Options::from(&config);
         Arc::new(opt)
     }

tskv/src/kv_option.rs

@@ -49,6 +49,7 @@ pub struct StorageOptions {
     pub max_concurrent_compaction: u16,
     pub strict_write: bool,
     pub snapshot_holding_time: i64,
+    pub max_datablock_size: u64,
 }
 
 // database/data/ts_family_id/tsm
@@ -105,6 +106,7 @@ impl From<&Config> for StorageOptions {
             max_concurrent_compaction: config.storage.max_concurrent_compaction,
             strict_write: config.storage.strict_write,
             snapshot_holding_time: config.cluster.snapshot_holding_time.as_secs() as i64,
+            max_datablock_size: config.storage.max_datablock_size,
         }
     }
 }

tskv/src/tseries_family.rs

@@ -873,7 +873,6 @@ pub struct TseriesFamily {
 }
 
 impl TseriesFamily {
-    pub const MAX_DATA_BLOCK_SIZE: u32 = 1000;
     #[allow(clippy::too_many_arguments)]
     #[cfg(test)]
     pub fn new(