Skip to content

Commit c3de50f

Browse files
committed
feat(storage): increase blob file size
1 parent c4ea2aa commit c3de50f

File tree

1 file changed

+46
-8
lines changed

1 file changed

+46
-8
lines changed

storage/src/db/kv_db/tables.rs

Lines changed: 46 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ use super::refcount;
1010
// took from
1111
// https://github.com/tikv/tikv/blob/d60c7fb6f3657dc5f3c83b0e3fc6ac75636e1a48/src/config/mod.rs#L170
1212
// todo: need to benchmark and update if it's not optimal
13+
const DEFAULT_BLOB_FILE_SIZE: u64 = bytesize::GIB;
1314
const DEFAULT_MIN_BLOB_SIZE: u64 = bytesize::KIB * 32;
1415

1516
/// Stores generic node parameters
@@ -49,7 +50,12 @@ impl ColumnFamilyOptions<Caches> for ArchiveBlockIds {
4950
opts.set_merge_operator_associative("archive_data_merge", archive_data_merge);
5051
// data is hardly compressible and dataset is small
5152
opts.set_compression_type(DBCompressionType::None);
52-
with_blob_db(opts, DEFAULT_MIN_BLOB_SIZE, DBCompressionType::None);
53+
with_blob_db(
54+
opts,
55+
DEFAULT_BLOB_FILE_SIZE,
56+
DEFAULT_MIN_BLOB_SIZE,
57+
DBCompressionType::None,
58+
);
5359
}
5460
}
5561

@@ -73,7 +79,12 @@ impl ColumnFamilyOptions<Caches> for Archives {
7379

7480
// data is already compressed
7581
opts.set_compression_type(DBCompressionType::None);
76-
with_blob_db(opts, DEFAULT_MIN_BLOB_SIZE, DBCompressionType::None);
82+
with_blob_db(
83+
opts,
84+
DEFAULT_BLOB_FILE_SIZE,
85+
DEFAULT_MIN_BLOB_SIZE,
86+
DBCompressionType::None,
87+
);
7788

7889
opts.set_max_write_buffer_number(8); // 8 * 512MB = 4GB;
7990
opts.set_write_buffer_size(512 * 1024 * 1024); // 512 per memtable
@@ -165,7 +176,12 @@ impl ColumnFamilyOptions<Caches> for PackageEntries {
165176
opts.set_write_buffer_size(512 * 1024 * 1024); // 512 per memtable
166177
opts.set_min_write_buffer_number_to_merge(2); // allow early flush
167178

168-
with_blob_db(opts, DEFAULT_MIN_BLOB_SIZE, DBCompressionType::Zstd);
179+
with_blob_db(
180+
opts,
181+
DEFAULT_BLOB_FILE_SIZE,
182+
DEFAULT_MIN_BLOB_SIZE,
183+
DBCompressionType::Zstd,
184+
);
169185

170186
// This flag specifies that the implementation should optimize the filters
171187
// mainly for cases where keys are found rather than also optimize for keys
@@ -204,7 +220,12 @@ impl ColumnFamilyOptions<Caches> for BlockDataEntries {
204220

205221
// data is already compressed
206222
opts.set_compression_type(DBCompressionType::None);
207-
with_blob_db(opts, DEFAULT_MIN_BLOB_SIZE, DBCompressionType::None);
223+
with_blob_db(
224+
opts,
225+
DEFAULT_BLOB_FILE_SIZE,
226+
DEFAULT_MIN_BLOB_SIZE,
227+
DBCompressionType::None,
228+
);
208229
}
209230
}
210231

@@ -405,7 +426,12 @@ impl ColumnFamily for ShardInternalMessages {
405426
impl ColumnFamilyOptions<Caches> for ShardInternalMessages {
406427
fn options(opts: &mut Options, caches: &mut Caches) {
407428
internal_queue_options(opts, caches);
408-
with_blob_db(opts, DEFAULT_MIN_BLOB_SIZE, DBCompressionType::None);
429+
with_blob_db(
430+
opts,
431+
DEFAULT_BLOB_FILE_SIZE,
432+
DEFAULT_MIN_BLOB_SIZE,
433+
DBCompressionType::None,
434+
);
409435
}
410436
}
411437

@@ -665,7 +691,12 @@ impl ColumnFamilyOptions<Caches> for Transactions {
665691
fn options(opts: &mut Options, caches: &mut Caches) {
666692
zstd_block_based_table_factory(opts, caches);
667693
opts.set_compression_type(DBCompressionType::Zstd);
668-
with_blob_db(opts, DEFAULT_MIN_BLOB_SIZE, DBCompressionType::Zstd);
694+
with_blob_db(
695+
opts,
696+
DEFAULT_BLOB_FILE_SIZE,
697+
DEFAULT_MIN_BLOB_SIZE,
698+
DBCompressionType::Zstd,
699+
);
669700
}
670701
}
671702

@@ -807,10 +838,17 @@ fn zstd_block_based_table_factory(opts: &mut Options, caches: &Caches) {
807838
opts.set_compression_type(DBCompressionType::Zstd);
808839
}
809840

810-
fn with_blob_db(opts: &mut Options, min_value_size: u64, compression_type: DBCompressionType) {
841+
fn with_blob_db(
842+
opts: &mut Options,
843+
file_size: u64,
844+
min_value_size: u64,
845+
compression_type: DBCompressionType,
846+
) {
811847
opts.set_enable_blob_files(true);
848+
opts.set_blob_file_size(file_size);
849+
opts.set_min_blob_size(min_value_size);
850+
812851
opts.set_enable_blob_gc(true);
813852

814-
opts.set_min_blob_size(min_value_size);
815853
opts.set_blob_compression_type(compression_type);
816854
}

0 commit comments

Comments
 (0)